From f29cdf6a2de53329e54ddd405bbc40bf3345de60 Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Tue, 31 May 2022 15:46:17 -0400 Subject: [PATCH 1/5] naive aar support --- maven/JarAssembler.kt | 172 ++++++++++++++++++++++++++++++++++-------- maven/PomGenerator.kt | 9 +++ maven/rules.bzl | 60 ++++++++++++--- 3 files changed, 202 insertions(+), 39 deletions(-) diff --git a/maven/JarAssembler.kt b/maven/JarAssembler.kt index cdd4b5e3..9c323ba4 100644 --- a/maven/JarAssembler.kt +++ b/maven/JarAssembler.kt @@ -28,14 +28,15 @@ import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.File import java.io.FileOutputStream -import java.lang.RuntimeException import java.nio.charset.Charset import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.Callable import java.util.zip.ZipEntry import java.util.zip.ZipFile +import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream +import kotlin.RuntimeException import kotlin.collections.HashMap import kotlin.system.exitProcess @@ -58,48 +59,159 @@ class JarAssembler : Callable { @Option(names = ["--jars"], split = ";") lateinit var jars: Array - private val entries = HashMap() - private val entryNames = mutableSetOf() + private val isAarPackaging get() = outputFile.extension == "aar" override fun call() { - ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { out -> - if (pomFile != null) { - val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" - entries += preCreateDirectories(Paths.get(pomPath)) - entries[pomPath] = pomFile!!.readBytes() + when (isAarPackaging) { + true -> assembleAar() + else -> assembleJar() + } + } + + // TODO: Call into to build class jar for aar + private fun assembleJar() { + val entries = hashMapOf() + val entryNames = mutableSetOf() + + if (pomFile != null) { + val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" + entries += preCreateDirectories(Paths.get(pomPath)) + entries[pomPath] = pomFile!!.readBytes() + } + + for (jar in jars) { + if (jar.extension == "aar") { + throw RuntimeException("cannot package AAR unless output is AAR") } - for (jar in jars) { - ZipFile(jar).use { jarZip -> - jarZip.entries().asSequence().forEach { entry -> - if (entryNames.contains(entry.name)) { - throw RuntimeException("duplicate entry in the JAR: ${entry.name}") - } - if (entry.name.contains("META-INF")) { - // pom.xml will be added by us - return@forEach - } - if (entry.isDirectory) { - // needed directories would be added by us - return@forEach - } - entryNames.add(entry.name) - BufferedInputStream(jarZip.getInputStream(entry)).use { inputStream -> - val sourceFileBytes = inputStream.readBytes() - val resultLocation = getFinalPath(entry, sourceFileBytes) - entries += preCreateDirectories(Paths.get(resultLocation)) - entries[resultLocation] = sourceFileBytes + ZipFile(jar).use { jarZip -> + jarZip.entries().asSequence().forEach { entry -> + if (entryNames.contains(entry.name)) { + return@forEach + // throw RuntimeException("duplicate entry in the JAR: ${entry.name}") + } + if (entry.name.contains("META-INF/maven")) { + // pom.xml will be added by us + return@forEach + } + if (entry.isDirectory) { + // needed directories would be added by us + return@forEach + } + entryNames.add(entry.name) + entries.processEntry(jarZip, entry) + } + } + } + + ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { jar -> + entries.keys.sorted().forEach { + val newEntry = ZipEntry(it) + jar.putNextEntry(newEntry) + jar.write(entries[it]!!) + } + } + } + + + private fun assembleAar() { + val entries = hashMapOf() + val entryNames = mutableSetOf() + + val classes = hashMapOf() + val classNames = mutableSetOf() + + if (pomFile != null) { + val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" + entries += preCreateDirectories(Paths.get(pomPath)) + entries[pomPath] = pomFile!!.readBytes() + } + + var baseAar = jars.single { it.extension == "aar" } + ZipFile(baseAar).use { aar -> + aar.entries().asSequence().forEach { entry -> + if (entry.name.contains("META-INF/maven")) { + // pom.xml will be added by us + } else if (entry.isDirectory) { + // needed directories would be added by us + } else if (entry.name == "classes.jar") { + // pull out classes in nested JAR + entry.let(aar::getInputStream).let(::ZipInputStream).use { classesJar -> + var zipEntry: ZipEntry? = classesJar.nextEntry + while (zipEntry != null) { + if (zipEntry.name.contains("META-INF/maven")) { + // pom.xml will be added by us + } else if (zipEntry.isDirectory) { + // needed directories would be added by us + } else { + classNames.add(zipEntry.name) + val sourceFileBytes = classesJar.readBytes() + val resultLocation = getFinalPath(zipEntry, sourceFileBytes) + classes += preCreateDirectories(Paths.get(resultLocation)) + classes[resultLocation] = sourceFileBytes + } + zipEntry = classesJar.nextEntry } } + } else { + // add to top-level entries + entryNames.add(entry.name) + entries.processEntry(aar, entry) } } + } + + // merge the rest of the class jars + for (jar in jars.filter { it.extension != "aar" }) { + ZipFile(jar).use { jarZip -> + jarZip.entries().asSequence().forEach { entry -> + if (classNames.contains(entry.name)) { + // TODO: Investigate why this is + println("I have a duplicate entry: ${entry.name}") + return@forEach +// throw RuntimeException("duplicate entry in the JAR: ${entry.name}") + } + if (entry.name.contains("META-INF/maven")) { + // pom.xml will be added by us + return@forEach + } + if (entry.isDirectory) { + // needed directories would be added by us + return@forEach + } + classNames.add(entry.name) + classes.processEntry(jarZip, entry) + } + } + } + + ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { aar -> + ZipEntry("classes.jar").let(aar::putNextEntry) + val classJar = ZipOutputStream(aar) + classes.keys.sorted().forEach { + val newEntry = ZipEntry(it) + classJar.putNextEntry(newEntry) + classJar.write(classes[it]) + } + classJar.finish() + entries.keys.sorted().forEach { val newEntry = ZipEntry(it) - out.putNextEntry(newEntry) - out.write(entries[it]!!) + aar.putNextEntry(newEntry) + aar.write(entries[it]!!) } } } + /** Add [ZipEntry] information to [this] map */ + private fun HashMap.processEntry(file: ZipFile, entry: ZipEntry) { + BufferedInputStream(file.getInputStream(entry)).use { inputStream -> + val sourceFileBytes = inputStream.readBytes() + val resultLocation = getFinalPath(entry, sourceFileBytes) + this += preCreateDirectories(Paths.get(resultLocation)) + this[resultLocation] = sourceFileBytes + } + } + /** * For path "a/b/c.java" inserts "a/" and "a/b/ into `entries` */ diff --git a/maven/PomGenerator.kt b/maven/PomGenerator.kt index 79b2b3f2..ffb985d1 100644 --- a/maven/PomGenerator.kt +++ b/maven/PomGenerator.kt @@ -74,6 +74,9 @@ class PomGenerator : Callable { @Option(names = ["--target_deps_coordinates"]) lateinit var dependencyCoordinates: String + @Option(names = ["--packaging"]) + var packaging = "" + fun getLicenseInfo(license_id: String): Pair { return when { license_id.equals("apache") -> { @@ -224,6 +227,12 @@ class PomGenerator : Callable { versionElem.appendChild(pom.createTextNode(version)) rootElement.appendChild(versionElem) + if (packaging.isNotEmpty() && packaging != "jar") { + val packagingElem = pom.createElement("packaging") + packagingElem.appendChild(pom.createTextNode(packaging)) + rootElement.appendChild(packagingElem) + } + // add dependency information rootElement.appendChild(dependencies(pom, version, workspace_refs)) diff --git a/maven/rules.bzl b/maven/rules.bzl index 52be4e69..021a96e9 100644 --- a/maven/rules.bzl +++ b/maven/rules.bzl @@ -17,6 +17,13 @@ # under the License. # +DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS = [ + Label("@bazel_tools//tools/android:android_jar"), + Label("@grab_bazel_common//tools/android:android_sdk"), +] + +def _is_android_library(target): + return AndroidLibraryAarInfo in target def _parse_maven_coordinates(coordinates_string, enforce_version_template=True): coordinates = coordinates_string.split(':') @@ -47,13 +54,14 @@ def _generate_version_file(ctx): def _generate_pom_file(ctx, version_file): target = ctx.attr.target - maven_coordinates = _parse_maven_coordinates(target[JarInfo].name) + jar_info = target[JarInfo] + maven_coordinates = _parse_maven_coordinates(jar_info.name) pom_file = ctx.actions.declare_file("{}_pom.xml".format(ctx.attr.name)) pom_deps = [] - for pom_dependency in [dep for dep in target[JarInfo].deps.to_list() if dep.type == 'pom']: + for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom']: pom_dependency = pom_dependency.maven_coordinates - if pom_dependency == target[JarInfo].name: + if pom_dependency == jar_info.name: continue pom_dependency_coordinates = _parse_maven_coordinates(pom_dependency, False) pom_dependency_artifact = pom_dependency_coordinates.group_id + ":" + pom_dependency_coordinates.artifact_id @@ -78,6 +86,7 @@ def _generate_pom_file(ctx, version_file): "--version_file=" + version_file.path, "--output_file=" + pom_file.path, "--workspace_refs_file=" + ctx.file.workspace_refs.path, + "--packaging=" + jar_info.packaging ], ) @@ -88,12 +97,14 @@ def _generate_class_jar(ctx, pom_file): maven_coordinates = _parse_maven_coordinates(target[JarInfo].name) jar = None - if hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar": + if (_is_android_library(target)): + jar = target[AndroidLibraryAarInfo].aar + elif hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar": jar = target[JavaInfo].outputs.jars[0].class_jar else: fail("Could not find JAR file to deploy in {}".format(target)) - output_jar = ctx.actions.declare_file("{}:{}.jar".format(maven_coordinates.group_id, maven_coordinates.artifact_id)) + output_jar = ctx.actions.declare_file("{}:{}.{}".format(maven_coordinates.group_id, maven_coordinates.artifact_id, target[JarInfo].packaging)) class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar'] class_jar_paths = [jar.path] + [target.path for target in class_jar_deps] @@ -119,7 +130,18 @@ def _generate_source_jar(ctx): srcjar = None - if hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar": + # TODO: Fix Android sources + if (_is_android_library(target)): + # print("sources") + # print(target[JavaInfo].source_jars) + # srcjar = target[JavaInfo].source_jars[0] + for output in target[JavaInfo].outputs.jars: + if output.source_jar and (output.source_jar.basename.endswith("-src.jar") or output.source_jar.basename.endswith("-sources.jar")): + # print("found source jar: ") + # print(output.source_jar) + srcjar = output.source_jar + # break + elif hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar": for output in target[JavaInfo].outputs.jars: if output.source_jar and (output.source_jar.basename.endswith("-src.jar") or output.source_jar.basename.endswith("-sources.jar")): srcjar = output.source_jar @@ -176,6 +198,7 @@ JarInfo = provider( fields = { "name": "The name of a the JAR (Maven coordinates)", "deps": "The list of dependencies of this JAR. A dependency may be of two types, POM or JAR.", + "packaging": "The type of target to publish (jar, war, aar, etc.)" }, ) @@ -184,10 +207,11 @@ def _aggregate_dependency_info_impl(target, ctx): deps = getattr(ctx.rule.attr, "deps", []) runtime_deps = getattr(ctx.rule.attr, "runtime_deps", []) exports = getattr(ctx.rule.attr, "exports", []) - deps_all = deps + exports + runtime_deps + deps_all = [dep for dep in deps + exports + runtime_deps if dep.label not in DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS] maven_coordinates = find_maven_coordinates(target, tags) dependencies = [] + packaging = "aar" if _is_android_library(target) else "jar" # depend via POM if maven_coordinates: @@ -195,11 +219,19 @@ def _aggregate_dependency_info_impl(target, ctx): type = "pom", maven_coordinates = maven_coordinates )] + # Hacky way to ignore something we don't care about but not crash + elif target.label in DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS: + return JarInfo( + name = None, + deps = depset([]), + packaging = None, + ) # include runtime output jars - elif target[JavaInfo].runtime_output_jars: + elif JavaInfo in target: jars = target[JavaInfo].runtime_output_jars source_jars = target[JavaInfo].source_jars dependencies = [struct( + target = str(target.label), type = "jar", class_jar = jar, source_jar = source_jar, @@ -209,7 +241,7 @@ def _aggregate_dependency_info_impl(target, ctx): else: fail("Unsure how to package dependency for target: %s" % target) - return JarInfo( + jar_info = JarInfo( name = maven_coordinates, deps = depset(dependencies, transitive = [ # Filter transitive JARs from dependency that has maven coordinates @@ -218,8 +250,18 @@ def _aggregate_dependency_info_impl(target, ctx): depset([dep for dep in target[JarInfo].deps.to_list() if dep.type == 'pom']) if target[JarInfo].name else target[JarInfo].deps for target in deps_all ]), + packaging = packaging, ) + print("JarInfo for: %s\n%s" % (target, json.encode_indent(dict( + target = str(target.label), + maven = jar_info.name, + deps = str(jar_info.deps.to_list()), + packaging = jar_info.packaging, + )))) + + return jar_info + aggregate_dependency_info = aspect( attr_aspects = [ "jars", From 4f05c07e379d90595e9b8e21093d48453c31404b Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Tue, 31 May 2022 23:55:18 -0400 Subject: [PATCH 2/5] add packaging to deploy script --- maven/rules.bzl | 14 +++++--------- maven/templates/deploy.py | 10 ++++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/maven/rules.bzl b/maven/rules.bzl index 021a96e9..0991897b 100644 --- a/maven/rules.bzl +++ b/maven/rules.bzl @@ -181,7 +181,7 @@ def _assemble_maven_impl(ctx): return [ DefaultInfo(files = depset(output_files)), - MavenDeploymentInfo(jar = class_jar, pom = pom_file, srcjar = source_jar) + MavenDeploymentInfo(packaging = ctx.attr.target[JarInfo].packaging, jar = class_jar, pom = pom_file, srcjar = source_jar) ] def find_maven_coordinates(target, tags): @@ -253,13 +253,6 @@ def _aggregate_dependency_info_impl(target, ctx): packaging = packaging, ) - print("JarInfo for: %s\n%s" % (target, json.encode_indent(dict( - target = str(target.label), - maven = jar_info.name, - deps = str(jar_info.deps.to_list()), - packaging = jar_info.packaging, - )))) - return jar_info aggregate_dependency_info = aspect( @@ -343,6 +336,7 @@ assemble_maven = rule( MavenDeploymentInfo = provider( fields = { + 'packaging': 'The type of target to publish (jar, war, aar, etc.)', 'jar': 'JAR file to deploy', 'srcjar': 'JAR file with sources', 'pom': 'Accompanying pom.xml file' @@ -356,6 +350,7 @@ def _deploy_maven_impl(ctx): lib_jar_link = "lib.jar" src_jar_link = "lib.srcjar" pom_xml_link = ctx.attr.target[MavenDeploymentInfo].pom.basename + packaging = ctx.attr.target[MavenDeploymentInfo].packaging ctx.actions.expand_template( template = ctx.file._deployment_script, @@ -365,7 +360,8 @@ def _deploy_maven_impl(ctx): "$SRCJAR_PATH": src_jar_link, "$POM_PATH": pom_xml_link, "{snapshot}": ctx.attr.snapshot, - "{release}": ctx.attr.release + "{release}": ctx.attr.release, + "$PACKAGING": packaging, } ) diff --git a/maven/templates/deploy.py b/maven/templates/deploy.py index a1f7218d..eff57d3e 100644 --- a/maven/templates/deploy.py +++ b/maven/templates/deploy.py @@ -95,6 +95,7 @@ def unpack_args(_, a, b=False): jar_path = "$JAR_PATH" pom_file_path = "$POM_PATH" srcjar_path = "$SRCJAR_PATH" +packaging = "$PACKAGING" namespace = { 'namespace': 'http://maven.apache.org/POM/4.0.0' } root = ElementTree.parse(pom_file_path).getroot() @@ -127,12 +128,13 @@ def unpack_args(_, a, b=False): 'must have a version which complies to this regex: {}' .format(version, repo_type, version_release_regex)) +artifact_extension = '.' + packaging filename_base = '{coordinates}/{artifact}/{version}/{artifact}-{version}'.format( coordinates=group_id.text.replace('.', '/'), version=version, artifact=artifact_id.text) -upload(maven_url, username, password, jar_path, filename_base + '.jar') +upload(maven_url, username, password, jar_path, filename_base + artifact_extension) if should_sign: - upload(maven_url, username, password, sign(jar_path), filename_base + '.jar.asc') + upload(maven_url, username, password, sign(jar_path), filename_base + artifact_extension + '.asc') upload(maven_url, username, password, pom_file_path, filename_base + '.pom') if should_sign: upload(maven_url, username, password, sign(pom_file_path), filename_base + '.pom.asc') @@ -158,12 +160,12 @@ def unpack_args(_, a, b=False): with tempfile.NamedTemporaryFile(mode='wt', delete=True) as jar_md5: jar_md5.write(md5(jar_path)) jar_md5.flush() - upload(maven_url, username, password, jar_md5.name, filename_base + '.jar.md5') + upload(maven_url, username, password, jar_md5.name, filename_base + artifact_extension + '.md5') with tempfile.NamedTemporaryFile(mode='wt', delete=True) as jar_sha1: jar_sha1.write(sha1(jar_path)) jar_sha1.flush() - upload(maven_url, username, password, jar_sha1.name, filename_base + '.jar.sha1') + upload(maven_url, username, password, jar_sha1.name, filename_base + artifact_extension + '.sha1') if os.path.exists(srcjar_path): with tempfile.NamedTemporaryFile(mode='wt', delete=True) as srcjar_md5: From 4bcfe8b71c16ad61c088d61ad161f309ce2c4584 Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Wed, 1 Jun 2022 12:45:32 -0400 Subject: [PATCH 3/5] move exclusions to assemble target attribute --- maven/rules.bzl | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/maven/rules.bzl b/maven/rules.bzl index 0991897b..a453026a 100644 --- a/maven/rules.bzl +++ b/maven/rules.bzl @@ -17,9 +17,8 @@ # under the License. # -DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS = [ +_DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS = [ Label("@bazel_tools//tools/android:android_jar"), - Label("@grab_bazel_common//tools/android:android_sdk"), ] def _is_android_library(target): @@ -59,7 +58,7 @@ def _generate_pom_file(ctx, version_file): pom_file = ctx.actions.declare_file("{}_pom.xml".format(ctx.attr.name)) pom_deps = [] - for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom']: + for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom' and dep.target not in ctx.attr.exclusions]: pom_dependency = pom_dependency.maven_coordinates if pom_dependency == jar_info.name: continue @@ -106,7 +105,7 @@ def _generate_class_jar(ctx, pom_file): output_jar = ctx.actions.declare_file("{}:{}.{}".format(maven_coordinates.group_id, maven_coordinates.artifact_id, target[JarInfo].packaging)) - class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar'] + class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.target not in ctx.attr.exclusions] class_jar_paths = [jar.path] + [target.path for target in class_jar_deps] ctx.actions.run( @@ -130,18 +129,7 @@ def _generate_source_jar(ctx): srcjar = None - # TODO: Fix Android sources - if (_is_android_library(target)): - # print("sources") - # print(target[JavaInfo].source_jars) - # srcjar = target[JavaInfo].source_jars[0] - for output in target[JavaInfo].outputs.jars: - if output.source_jar and (output.source_jar.basename.endswith("-src.jar") or output.source_jar.basename.endswith("-sources.jar")): - # print("found source jar: ") - # print(output.source_jar) - srcjar = output.source_jar - # break - elif hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar": + if _is_android_library(target) or (hasattr(target, "files") and target.files.to_list() and target.files.to_list()[0].extension == "jar"): for output in target[JavaInfo].outputs.jars: if output.source_jar and (output.source_jar.basename.endswith("-src.jar") or output.source_jar.basename.endswith("-sources.jar")): srcjar = output.source_jar @@ -154,7 +142,7 @@ def _generate_source_jar(ctx): output_jar = ctx.actions.declare_file("{}:{}-sources.jar".format(maven_coordinates.group_id, maven_coordinates.artifact_id)) - source_jar_deps = [dep.source_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.source_jar] + source_jar_deps = [dep.source_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.source_jar and dep.target not in ctx.attr.exclusions] source_jar_paths = [srcjar.path] + [target.path for target in source_jar_deps] ctx.actions.run( @@ -207,7 +195,7 @@ def _aggregate_dependency_info_impl(target, ctx): deps = getattr(ctx.rule.attr, "deps", []) runtime_deps = getattr(ctx.rule.attr, "runtime_deps", []) exports = getattr(ctx.rule.attr, "exports", []) - deps_all = [dep for dep in deps + exports + runtime_deps if dep.label not in DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS] + deps_all = deps + exports + runtime_deps maven_coordinates = find_maven_coordinates(target, tags) dependencies = [] @@ -220,7 +208,7 @@ def _aggregate_dependency_info_impl(target, ctx): maven_coordinates = maven_coordinates )] # Hacky way to ignore something we don't care about but not crash - elif target.label in DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS: + elif target.label in _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS: return JarInfo( name = None, deps = depset([]), @@ -231,7 +219,7 @@ def _aggregate_dependency_info_impl(target, ctx): jars = target[JavaInfo].runtime_output_jars source_jars = target[JavaInfo].source_jars dependencies = [struct( - target = str(target.label), + target = target, type = "jar", class_jar = jar, source_jar = source_jar, @@ -313,6 +301,10 @@ assemble_maven = rule( default = "PROJECT_URL", doc = "Project source control URL to fill into pom.xml", ), + "exclusions": attr.label_list( + default = _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS, + doc = "Labels to not capture in transitive closure when assembling the artifact" + ), "_pom_generator": attr.label( default = "@vaticle_bazel_distribution//maven:pom-generator", executable = True, From 5f8b345abfe8d78b9cff6a42301a79a34397d0ea Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Wed, 1 Jun 2022 13:31:40 -0400 Subject: [PATCH 4/5] use neverlink attribute over explicit exclusions --- maven/rules.bzl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/maven/rules.bzl b/maven/rules.bzl index a453026a..fe40c7a6 100644 --- a/maven/rules.bzl +++ b/maven/rules.bzl @@ -17,6 +17,7 @@ # under the License. # +# Known generic labels to automatically not include in closure _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS = [ Label("@bazel_tools//tools/android:android_jar"), ] @@ -58,7 +59,7 @@ def _generate_pom_file(ctx, version_file): pom_file = ctx.actions.declare_file("{}_pom.xml".format(ctx.attr.name)) pom_deps = [] - for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom' and dep.target not in ctx.attr.exclusions]: + for pom_dependency in [dep for dep in jar_info.deps.to_list() if dep.type == 'pom']: pom_dependency = pom_dependency.maven_coordinates if pom_dependency == jar_info.name: continue @@ -105,7 +106,7 @@ def _generate_class_jar(ctx, pom_file): output_jar = ctx.actions.declare_file("{}:{}.{}".format(maven_coordinates.group_id, maven_coordinates.artifact_id, target[JarInfo].packaging)) - class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.target not in ctx.attr.exclusions] + class_jar_deps = [dep.class_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar'] class_jar_paths = [jar.path] + [target.path for target in class_jar_deps] ctx.actions.run( @@ -142,7 +143,7 @@ def _generate_source_jar(ctx): output_jar = ctx.actions.declare_file("{}:{}-sources.jar".format(maven_coordinates.group_id, maven_coordinates.artifact_id)) - source_jar_deps = [dep.source_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.source_jar and dep.target not in ctx.attr.exclusions] + source_jar_deps = [dep.source_jar for dep in target[JarInfo].deps.to_list() if dep.type == 'jar' and dep.source_jar] source_jar_paths = [srcjar.path] + [target.path for target in source_jar_deps] ctx.actions.run( @@ -195,6 +196,7 @@ def _aggregate_dependency_info_impl(target, ctx): deps = getattr(ctx.rule.attr, "deps", []) runtime_deps = getattr(ctx.rule.attr, "runtime_deps", []) exports = getattr(ctx.rule.attr, "exports", []) + neverlink = getattr(ctx.rule.attr, "neverlink", False) deps_all = deps + exports + runtime_deps maven_coordinates = find_maven_coordinates(target, tags) @@ -204,11 +206,12 @@ def _aggregate_dependency_info_impl(target, ctx): # depend via POM if maven_coordinates: dependencies = [struct( + target = target, type = "pom", maven_coordinates = maven_coordinates )] # Hacky way to ignore something we don't care about but not crash - elif target.label in _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS: + elif neverlink or target.label in _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS: return JarInfo( name = None, deps = depset([]), @@ -301,10 +304,6 @@ assemble_maven = rule( default = "PROJECT_URL", doc = "Project source control URL to fill into pom.xml", ), - "exclusions": attr.label_list( - default = _DO_NOT_INCLUDE_IN_TRANSITIVE_CLOSURE_TARGETS, - doc = "Labels to not capture in transitive closure when assembling the artifact" - ), "_pom_generator": attr.label( default = "@vaticle_bazel_distribution//maven:pom-generator", executable = True, From 286edb982fbed85d55a3e8c12b00c79f6185bccd Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Wed, 1 Jun 2022 15:13:16 -0400 Subject: [PATCH 5/5] cleanup jar assembler --- maven/JarAssembler.kt | 196 ++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 120 deletions(-) diff --git a/maven/JarAssembler.kt b/maven/JarAssembler.kt index 9c323ba4..acde77f9 100644 --- a/maven/JarAssembler.kt +++ b/maven/JarAssembler.kt @@ -28,6 +28,7 @@ import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.File import java.io.FileOutputStream +import java.io.InputStream import java.nio.charset.Charset import java.nio.file.Path import java.nio.file.Paths @@ -37,9 +38,10 @@ import java.util.zip.ZipFile import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream import kotlin.RuntimeException -import kotlin.collections.HashMap import kotlin.system.exitProcess +typealias Entries = MutableMap +private fun Entries(): Entries = mutableMapOf() @Command(name = "jar-assembler", mixinStandardHelpOptions = true) class JarAssembler : Callable { @@ -59,164 +61,118 @@ class JarAssembler : Callable { @Option(names = ["--jars"], split = ";") lateinit var jars: Array - private val isAarPackaging get() = outputFile.extension == "aar" - override fun call() { - when (isAarPackaging) { - true -> assembleAar() - else -> assembleJar() - } - } - - // TODO: Call into to build class jar for aar - private fun assembleJar() { - val entries = hashMapOf() - val entryNames = mutableSetOf() + Entries().apply { + pomFile?.readBytes()?.let { pomContents -> + val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" + this += preCreateDirectories(Paths.get(pomPath)) + this[pomPath] = pomContents + } - if (pomFile != null) { - val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" - entries += preCreateDirectories(Paths.get(pomPath)) - entries[pomPath] = pomFile!!.readBytes() + ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { + if (outputFile.extension == "aar") assembleAar(it) + else assembleClassesJar(it) + } } + } + /** Assemble a class JAR containing the transitive class closure from [jars] and any pre-existing entries in [this] */ + private fun Entries.assembleClassesJar(output: ZipOutputStream, jars: List = this@JarAssembler.jars.toList()) { for (jar in jars) { if (jar.extension == "aar") { - throw RuntimeException("cannot package AAR unless output is AAR") - } - ZipFile(jar).use { jarZip -> - jarZip.entries().asSequence().forEach { entry -> - if (entryNames.contains(entry.name)) { - return@forEach - // throw RuntimeException("duplicate entry in the JAR: ${entry.name}") - } - if (entry.name.contains("META-INF/maven")) { - // pom.xml will be added by us - return@forEach - } - if (entry.isDirectory) { - // needed directories would be added by us - return@forEach - } - entryNames.add(entry.name) - entries.processEntry(jarZip, entry) - } + throw RuntimeException("cannot package AAR within classes JAR") } + processZip(ZipFile(jar)) } - ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { jar -> - entries.keys.sorted().forEach { - val newEntry = ZipEntry(it) - jar.putNextEntry(newEntry) - jar.write(entries[it]!!) - } - } + writeEntries(output) } - - private fun assembleAar() { - val entries = hashMapOf() - val entryNames = mutableSetOf() - - val classes = hashMapOf() - val classNames = mutableSetOf() - - if (pomFile != null) { - val pomPath = "META-INF/maven/${groupId}/${artifactId}/pom.xml" - entries += preCreateDirectories(Paths.get(pomPath)) - entries[pomPath] = pomFile!!.readBytes() - } - - var baseAar = jars.single { it.extension == "aar" } - ZipFile(baseAar).use { aar -> - aar.entries().asSequence().forEach { entry -> - if (entry.name.contains("META-INF/maven")) { - // pom.xml will be added by us - } else if (entry.isDirectory) { - // needed directories would be added by us - } else if (entry.name == "classes.jar") { + /** Assemble an AAR from a base AAR containing the transitive class closure from the additional [jars] and any pre-existing entries in [this] */ + private fun Entries.assembleAar(output: ZipOutputStream) { + val classes = Entries() + processZip(jars.single { it.extension == "aar" }.let(::ZipFile)) { aar, entry -> + validateEntry(entry)?.let { + if (entry.name == "classes.jar") { // pull out classes in nested JAR entry.let(aar::getInputStream).let(::ZipInputStream).use { classesJar -> var zipEntry: ZipEntry? = classesJar.nextEntry while (zipEntry != null) { - if (zipEntry.name.contains("META-INF/maven")) { - // pom.xml will be added by us - } else if (zipEntry.isDirectory) { - // needed directories would be added by us - } else { - classNames.add(zipEntry.name) - val sourceFileBytes = classesJar.readBytes() - val resultLocation = getFinalPath(zipEntry, sourceFileBytes) - classes += preCreateDirectories(Paths.get(resultLocation)) - classes[resultLocation] = sourceFileBytes - } + classes.processEntry(classesJar, zipEntry) zipEntry = classesJar.nextEntry } } } else { // add to top-level entries - entryNames.add(entry.name) - entries.processEntry(aar, entry) + processEntry(aar, entry) } } } - // merge the rest of the class jars - for (jar in jars.filter { it.extension != "aar" }) { - ZipFile(jar).use { jarZip -> - jarZip.entries().asSequence().forEach { entry -> - if (classNames.contains(entry.name)) { - // TODO: Investigate why this is - println("I have a duplicate entry: ${entry.name}") - return@forEach -// throw RuntimeException("duplicate entry in the JAR: ${entry.name}") - } - if (entry.name.contains("META-INF/maven")) { - // pom.xml will be added by us - return@forEach - } - if (entry.isDirectory) { - // needed directories would be added by us - return@forEach - } - classNames.add(entry.name) - classes.processEntry(jarZip, entry) - } - } - } + // write classes jar first + ZipEntry("classes.jar").let(output::putNextEntry) + val classJar = ZipOutputStream(output) + classes.assembleClassesJar(classJar, jars.filter { it.extension != "aar" }) + classJar.finish() - ZipOutputStream(BufferedOutputStream(FileOutputStream(outputFile))).use { aar -> - ZipEntry("classes.jar").let(aar::putNextEntry) - val classJar = ZipOutputStream(aar) - classes.keys.sorted().forEach { - val newEntry = ZipEntry(it) - classJar.putNextEntry(newEntry) - classJar.write(classes[it]) - } - classJar.finish() + // write the rest of the entries + writeEntries(output) + } - entries.keys.sorted().forEach { - val newEntry = ZipEntry(it) - aar.putNextEntry(newEntry) - aar.write(entries[it]!!) - } + /** [process] each [ZipEntry] in [file] within the context of [this] */ + private fun Entries.processZip(file: ZipFile, process: Entries.(zip: ZipFile, entry: ZipEntry) -> Unit = { zip, entry -> processEntry(zip, entry) }) = file.use { zip -> + zip.entries().asSequence().forEach { entry -> + process(zip, entry) } } - /** Add [ZipEntry] information to [this] map */ - private fun HashMap.processEntry(file: ZipFile, entry: ZipEntry) { - BufferedInputStream(file.getInputStream(entry)).use { inputStream -> + /** Validate [ZipEntry] and add information to [this] entries map */ + private fun Entries.processEntry(zip: ZipFile, entry: ZipEntry): Unit = BufferedInputStream(zip.getInputStream(entry)).use { + processEntry(it, entry) + } + + /** Validate [ZipEntry] and add information to [this] entries map */ + private fun Entries.processEntry(inputStream: InputStream, entry: ZipEntry) { + validateEntry(entry)?.let { val sourceFileBytes = inputStream.readBytes() - val resultLocation = getFinalPath(entry, sourceFileBytes) + val resultLocation = getFinalPath(it, sourceFileBytes) this += preCreateDirectories(Paths.get(resultLocation)) this[resultLocation] = sourceFileBytes } } + /** Return null if this [entry] shouldn't be processed */ + private fun Entries.validateEntry(entry: ZipEntry): ZipEntry? = when { + entry.isDirectory -> { + // needed directories would be added by us + null + } + entry.name.contains("META-INF/maven") -> { + // pom.xml will be added by us + null + } + keys.contains(entry.name) -> { + // TODO: Investigate why I'm getting duplicates + println("I have a duplicate entry: ${entry.name}") + null + // throw RuntimeException("duplicate entry in the JAR: ${entry.name}") + } + else -> entry + } + + /** Write entries captured in [this] to [output] */ + private fun Entries.writeEntries(output: ZipOutputStream) { + entries.sortedBy(Map.Entry::key).forEach { (key, entry) -> + output.putNextEntry(ZipEntry(key)) + output.write(entry) + } + } + /** * For path "a/b/c.java" inserts "a/" and "a/b/ into `entries` */ private fun preCreateDirectories(path: Path): Map { - val newEntries = HashMap() + val newEntries = Entries() for (i in path.nameCount-1 downTo 1) { val subPath = path.subpath(0, i).toString() + "/" newEntries[subPath] = ByteArray(0)