From 33ed9044708925c642f18be8de6f3aac21dea3f9 Mon Sep 17 00:00:00 2001 From: Romain Manni-Bucau Date: Fri, 29 May 2026 16:21:50 +0200 Subject: [PATCH] Rework janino scanner to make it lighter in mem and faster --- .../hop/core/plugins/HopURLClassLoader.java | 25 ++ .../core/vfs/HopVfsNetworkProvidersTest.java | 42 ++- plugins/transforms/janino/pom.xml | 5 + .../janino/function/FunctionLib.java | 83 +++-- .../janino/scanner/ClassLoaderScanner.java | 143 ++++++++ .../janino/scanner/JarExclusionsLoader.java | 74 ++++ .../ClassLoaderScanner.ignored-jars.txt | 343 ++++++++++++++++++ .../scanner/ClassLoaderScannerTest.java | 112 ++++++ .../janino/scanner/FunctionGenerator.java | 109 ++++++ .../janino/scanner/FunctionLibTest.java | 76 ++++ .../scanner/JarExclusionsLoaderTest.java | 78 ++++ 11 files changed, 1045 insertions(+), 45 deletions(-) create mode 100644 plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScanner.java create mode 100644 plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoader.java create mode 100644 plugins/transforms/janino/src/main/resources/ClassLoaderScanner.ignored-jars.txt create mode 100644 plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScannerTest.java create mode 100644 plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionGenerator.java create mode 100644 plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionLibTest.java create mode 100644 plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoaderTest.java diff --git a/core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java b/core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java index d1df3e390cf..999661c862e 100644 --- a/core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java +++ b/core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java @@ -21,10 +21,13 @@ import java.net.URL; import java.net.URLClassLoader; import java.security.ProtectionDomain; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; public class HopURLClassLoader extends URLClassLoader { private String name; + private final ConcurrentHashMap, Object> cache = new ConcurrentHashMap<>(); public HopURLClassLoader(URL[] url, ClassLoader classLoader) { super(url, classLoader); @@ -35,9 +38,31 @@ public HopURLClassLoader(URL[] url, ClassLoader classLoader, String name) { this.name = name; } + public T get(final Class key) { + return key.cast(cache.get(key)); + } + + public T computeIfAbsent(final Class key, Supplier provider) { + return key.cast(cache.computeIfAbsent(key, k -> provider)); + } + @Override protected void addURL(URL url) { super.addURL(url); + // invalidate the cache since it is wrong if related to the classloader, + // do not lock since we assume addURL is called in a safe context + cache.values().stream() + .filter(AutoCloseable.class::isInstance) + .map(AutoCloseable.class::cast) + .forEach( + it -> { + try { + it.close(); + } catch (final Exception e) { + // no-op + } + }); + cache.clear(); } @Override diff --git a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java index 928730fba12..c5c2c8385f8 100644 --- a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java +++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java @@ -17,6 +17,7 @@ package org.apache.hop.core.vfs; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpsConfigurator; @@ -24,7 +25,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -70,6 +73,7 @@ class HopVfsNetworkProvidersTest { private static final String FTP_PASS = "secret"; private static final String SFTP_USER = "alice"; private static final String SFTP_PASS = "secret"; + private static String LOCALHOST; @TempDir static Path sharedRoot; @@ -93,19 +97,27 @@ class HopVfsNetworkProvidersTest { private static SshServer sshServer; private static int sftpPort; + static { + try { + LOCALHOST = InetAddress.getLocalHost().getHostAddress(); + } catch (final UnknownHostException e) { + fail(e); + } + } + @BeforeAll static void startServers() throws Exception { keyStorePath = generateTestKeyStore(); // HTTP - httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + httpServer = HttpServer.create(new InetSocketAddress(LOCALHOST, 0), 0); httpPort = httpServer.getAddress().getPort(); httpServer.createContext("/payload.txt", new FixedPayloadHandler("http-payload")); httpServer.start(); // HTTPS SSLContext sslContext = buildServerSslContext(keyStorePath); - httpsServer = HttpsServer.create(new InetSocketAddress("localhost", 0), 0); + httpsServer = HttpsServer.create(new InetSocketAddress(LOCALHOST, 0), 0); httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext)); httpsServer.createContext("/secure.txt", new FixedPayloadHandler("https-payload")); httpsServer.start(); @@ -158,19 +170,22 @@ static void stopServers() throws Exception { @Test @DisplayName("http:// fetches a payload from an embedded HttpServer") void httpProviderReadsFromEmbeddedServer() throws Exception { - assertEquals("http-payload", readToString("http://localhost:" + httpPort + "/payload.txt")); + assertEquals( + "http-payload", readToString("http://" + LOCALHOST + ":" + httpPort + "/payload.txt")); } @Test @DisplayName("https:// fetches a payload over TLS from an embedded HttpsServer") void httpsProviderReadsFromEmbeddedServer() throws Exception { - assertEquals("https-payload", readToString("https://localhost:" + httpsPort + "/secure.txt")); + assertEquals( + "https-payload", readToString("https://" + LOCALHOST + ":" + httpsPort + "/secure.txt")); } @Test @DisplayName("ftp:// fetches a payload from an embedded Apache FtpServer") void ftpProviderReadsFromEmbeddedServer() throws Exception { - String url = "ftp://" + FTP_USER + ":" + FTP_PASS + "@localhost:" + ftpPort + "/greeting.txt"; + String url = + "ftp://" + FTP_USER + ":" + FTP_PASS + "@" + LOCALHOST + ":" + ftpPort + "/greeting.txt"; FileSystemOptions opts = new FileSystemOptions(); FtpFileSystemConfigBuilder.getInstance().setPassiveMode(opts, true); assertEquals("ftp-payload", readWithOptions(url, opts)); @@ -179,7 +194,8 @@ void ftpProviderReadsFromEmbeddedServer() throws Exception { @Test @DisplayName("ftps:// fetches a payload over TLS from an embedded Apache FtpServer") void ftpsProviderReadsFromEmbeddedServer() throws Exception { - String url = "ftps://" + FTP_USER + ":" + FTP_PASS + "@localhost:" + ftpsPort + "/greeting.txt"; + String url = + "ftps://" + FTP_USER + ":" + FTP_PASS + "@" + LOCALHOST + ":" + ftpsPort + "/greeting.txt"; FileSystemOptions opts = new FileSystemOptions(); FtpsFileSystemConfigBuilder ftps = FtpsFileSystemConfigBuilder.getInstance(); ftps.setPassiveMode(opts, true); @@ -191,7 +207,15 @@ void ftpsProviderReadsFromEmbeddedServer() throws Exception { @DisplayName("sftp:// fetches a payload from an embedded Apache MINA SSHD server") void sftpProviderReadsFromEmbeddedServer() throws Exception { String url = - "sftp://" + SFTP_USER + ":" + SFTP_PASS + "@localhost:" + sftpPort + "/greeting.txt"; + "sftp://" + + SFTP_USER + + ":" + + SFTP_PASS + + "@" + + LOCALHOST + + ":" + + sftpPort + + "/greeting.txt"; assertEquals("sftp-payload", readToString(url)); } @@ -238,7 +262,7 @@ private static Path generateTestKeyStore() throws Exception { "-dname", "CN=localhost, OU=Hop, O=Apache, L=Test, S=Test, C=US", "-ext", - "SAN=DNS:localhost,IP:127.0.0.1", + "SAN=DNS:localhost,IP:" + LOCALHOST, "-noprompt") .redirectErrorStream(true) .start(); @@ -302,7 +326,7 @@ private static FtpServerStart startFtp(Path home, boolean tls) throws Exception private static SshServer startSftp(Path home) throws IOException { SshServer sshd = SshServer.setUpDefaultServer(); - sshd.setHost("localhost"); + sshd.setHost(LOCALHOST); sshd.setPort(0); sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(sharedRoot.resolve("hostkey.ser"))); sshd.setPasswordAuthenticator(AcceptAllPasswordAuthenticator.INSTANCE); diff --git a/plugins/transforms/janino/pom.xml b/plugins/transforms/janino/pom.xml index f1a5ca59d9d..7c38d76d9e0 100644 --- a/plugins/transforms/janino/pom.xml +++ b/plugins/transforms/janino/pom.xml @@ -39,6 +39,11 @@ hop-transform-rowgenerator ${project.version} + + org.apache.xbean + xbean-finder-shaded + 4.30 + org.codehaus.janino janino diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/function/FunctionLib.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/function/FunctionLib.java index 37e4072725b..a2fd8ae861d 100644 --- a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/function/FunctionLib.java +++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/function/FunctionLib.java @@ -31,13 +31,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.plugins.HopURLClassLoader; import org.apache.hop.core.plugins.IPlugin; import org.apache.hop.core.plugins.PluginRegistry; import org.apache.hop.core.plugins.TransformPluginType; import org.apache.hop.core.util.Utils; +import org.apache.hop.pipeline.transforms.janino.scanner.ClassLoaderScanner; public class FunctionLib { - private List functions; public FunctionLib() throws HopException { @@ -46,49 +47,56 @@ public FunctionLib() throws HopException { PluginRegistry registry = PluginRegistry.getInstance(); IPlugin plugin = registry.getPlugin(TransformPluginType.class, "Janino"); ClassLoader loader = registry.getClassLoader(plugin); - Set> classes = - findAllClassesUsingGoogleGuice(loader, "org.apache.hop.pipeline.transforms.janino"); - - for (Class clazz : classes) { - Method[] methods = clazz.getMethods(); - for (Method method : methods) { - JaninoFunction annotation = method.getAnnotation(JaninoFunction.class); - List functionExamples = new ArrayList<>(); - if (annotation != null) { - if (!Utils.isEmpty(annotation.examples())) { - ObjectMapper mapper = new ObjectMapper(); - JsonNode arrayNode = mapper.readTree(annotation.examples()); - for (JsonNode jsonNode : arrayNode) { - functionExamples.add( - new FunctionExample( - jsonNode.get("expression").asText(), - jsonNode.get("result").asText(), - jsonNode.get("level").asText(), - jsonNode.get("comment").asText())); - } - } - - FunctionDescription functionDescription = - new FunctionDescription( - annotation.category(), - annotation.name(), - annotation.description(), - annotation.syntax(), - annotation.returns(), - null, - annotation.semantics(), - clazz.getCanonicalName(), - functionExamples); - functions.add(functionDescription); - } + + if (loader instanceof HopURLClassLoader hucl) { + var cached = hucl.get(CachedFunctions.class); + if (cached != null) { + functions.addAll(cached.functions()); + return; } } + doScan(loader); } catch (Exception e) { throw new HopException(e); } } + private void doScan(ClassLoader loader) throws IOException { + for (Method method : + new ClassLoaderScanner() + .findMethodsWithAnnotationInPackage( + loader, "org.apache.hop.pipeline.transforms.janino", JaninoFunction.class)) { + JaninoFunction annotation = method.getAnnotation(JaninoFunction.class); + List functionExamples = new ArrayList<>(); + if (!Utils.isEmpty(annotation.examples())) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode arrayNode = mapper.readTree(annotation.examples()); + for (JsonNode jsonNode : arrayNode) { + functionExamples.add( + new FunctionExample( + jsonNode.get("expression").asText(), + jsonNode.get("result").asText(), + jsonNode.get("level").asText(), + jsonNode.get("comment").asText())); + } + } + + FunctionDescription functionDescription = + new FunctionDescription( + annotation.category(), + annotation.name(), + annotation.description(), + annotation.syntax(), + annotation.returns(), + null, + annotation.semantics(), + method.getDeclaringClass().getCanonicalName(), + functionExamples); + functions.add(functionDescription); + } + } + /** * @return the functions */ @@ -172,6 +180,7 @@ public FunctionDescription getFunctionDescription(String functionName) { return null; } + @Deprecated // shouldn't be public, kept for legacy and external usage public Set> findAllClassesUsingGoogleGuice(ClassLoader classLoader, String packageName) throws IOException { return ClassPath.from(classLoader).getAllClasses().stream() @@ -188,4 +197,6 @@ public Set> findAllClassesUsingGoogleGuice(ClassLoader classLoader, Str }) .collect(Collectors.toSet()); } + + private record CachedFunctions(List functions) {} } diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScanner.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScanner.java new file mode 100644 index 00000000000..5f41550f68c --- /dev/null +++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScanner.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import java.io.File; +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.Collection; +import java.util.function.Predicate; +import java.util.jar.JarFile; +import java.util.stream.Stream; +import org.apache.xbean.finder.AnnotationFinder; +import org.apache.xbean.finder.ClassLoaders; +import org.apache.xbean.finder.archive.Archive; +import org.apache.xbean.finder.archive.CompositeArchive; +import org.apache.xbean.finder.archive.FileArchive; +import org.apache.xbean.finder.archive.FilteredArchive; +import org.apache.xbean.finder.archive.JarArchive; +import org.apache.xbean.finder.filter.Filters; + +// note: for now manifest.mf classpath are not handled, todo: review if needed (unlikely normally) +public final class ClassLoaderScanner { + public Collection findMethodsWithAnnotationInPackage( + ClassLoader classLoader, String packageName, Class annotation) + throws IOException { + final var urls = ClassLoaders.findUrls(classLoader); + // we cache the result of the scanning so no need to cache the exclusions - should save mem + final var jarFilter = new JarExclusionsLoader().load("ClassLoaderScanner.ignored-jars.txt"); + + // next line would work but doesn't cut fast enough the jar selection + // and requires a refiltering and iterating over all entries for a poor - almost always 0 - gain + // final var archives = ClasspathArchive.archives(classLoader, urls); + final var archives = + urls.stream() + .flatMap( + it -> + switch (it.getProtocol()) { + case "jar" -> { + final var archive = new JarArchive(classLoader, it); + yield !jarHasPackage(archive.getUrl(), jarFilter, packageName) + ? Stream.empty() + : Stream.of(archive); + } + case "file" -> { + try { // validate it is a jar else fallback on plain directory + final var jarUrl = new URL("jar", "", it.toExternalForm() + "!/"); + final var juc = (JarURLConnection) jarUrl.openConnection(); + juc.getJarFile(); + final var archive = new JarArchive(classLoader, jarUrl); + yield !jarHasPackage(archive.getUrl(), jarFilter, packageName) + ? Stream.empty() + : Stream.of(archive); + } catch (final IOException e) { + final var archive = new FileArchive(classLoader, it); + yield !dirHasPackage(archive.getDir(), packageName) + ? Stream.empty() + : Stream.of(archive); + } + } + default -> Stream.empty(); + }) + .toList(); + + final var aggregated = + new FilteredArchive( + archives.size() == 1 ? archives.getFirst() : new CompositeArchive(archives), + Filters.packages(packageName)); + try { + // todo: review if we try to check if META-INF/jandex.idx exists in the jar/dir + // and use it instead of doing a bytecode scanning with asm + // -> requires dev to use jandex but can be more consistent with plugins + // -> likely better: drop all that and do scanning at build time with ServiceLoader + // registration using a maven plugin or CLI to generate the right metadata + return new AnnotationFinder(aggregated, true).findAnnotatedMethods(annotation); + } finally { + if (aggregated instanceof AutoCloseable a) { // Composite/Jar archives + try { + a.close(); + } catch (Exception e) { + // no-op, ignored + } + } + } + } + + private boolean jarHasPackage(URL location, Predicate jarFilter, String packageName) { + File jarFile = null; + var idx = 0; + try { + var jarPath = + FileArchive.decode( + "jar".equalsIgnoreCase(location.getProtocol()) + ? new URL( + location.getPath().endsWith("!/") + ? location + .getPath() + .substring(0, location.getPath().lastIndexOf("!/")) + : location.getPath()) + .getPath() + : location.getPath()); + for (var jp = jarPath; + !(jarFile = new File(jp)).exists() && (idx = jarPath.indexOf("!/", idx + 1)) > 0; + jp = jarPath.substring(0, idx)) {} + try (final var jar = new JarFile(jarFile)) { + // todo: enhance with mjar support but so unlikely that we don't care for now + final var hasPck = jar.getEntry(packageName.replace('.', '/') + '/') != null; + if (!hasPck) { + return false; + } + + // if excluded with our default rules try to match an explicit marker + // this is a marker file enabling to force the jar scan even if excluded by default + if (!jarFilter.test(jarFile.getName())) { + return jar.getEntry("META-INF/org.apache.hop.janino") != null; + } + return true; + } + } catch (IOException e) { + return false; + } + } + + private boolean dirHasPackage(File location, String packageName) { + return new File(location, packageName.replace('.', '/')).exists(); + } +} diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoader.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoader.java new file mode 100644 index 00000000000..092449e7ff3 --- /dev/null +++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoader.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import static java.util.Optional.ofNullable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.function.Predicate; +import org.apache.xbean.finder.filter.Filters; + +public class JarExclusionsLoader { + public Predicate load(String resourcePath) { + try (var is = + ofNullable(Thread.currentThread().getContextClassLoader()) + .orElseGet(ClassLoader::getSystemClassLoader) + .getResourceAsStream(resourcePath)) { + if (is == null) { + throw new IllegalArgumentException("Resource not found: " + resourcePath); + } + return load(is); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public Predicate load(InputStream inputStream) { + final var ignoredPrefixes = new ArrayList(); + final var notIgnoredPrefixes = new ArrayList(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + if (line.startsWith("!")) { + notIgnoredPrefixes.add(line.substring(1)); + } else { + ignoredPrefixes.add(line); + } + } + if (notIgnoredPrefixes.isEmpty()) { + final var onlyIgnored = Filters.prefixes(ignoredPrefixes.toArray(new String[0])); + return p -> !onlyIgnored.accept(p); + } + final var ignored = Filters.prefixes(ignoredPrefixes.toArray(new String[0])); + final var notIgnored = Filters.prefixes(notIgnoredPrefixes.toArray(new String[0])); + return p -> notIgnored.accept(p) || !ignored.accept(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/plugins/transforms/janino/src/main/resources/ClassLoaderScanner.ignored-jars.txt b/plugins/transforms/janino/src/main/resources/ClassLoaderScanner.ignored-jars.txt new file mode 100644 index 00000000000..b232aaf491a --- /dev/null +++ b/plugins/transforms/janino/src/main/resources/ClassLoaderScanner.ignored-jars.txt @@ -0,0 +1,343 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +accessors-smart +adal4j +adapter-rxjava +aircompressor +airline +amazon-kinesis-client +angus- +animal-sniffer-annotations +annotations +antlr +antlr4- +aopalliance +apache-client +apache-mime4j- +api- +apiguardian- +apicurio- +Apple +args4j +arns +arrow- +asm +async- +auth +auto-common +automaton +auto- +avro +aws- +azure- +batik- +bcjmail- +bcprov- +beam- +bson +byte-buddy +caffeine +cassandra-all +checker-qual +chill_2.12 +chill-java +chronicle- +classgraph +clickhouse- +client +cloudwatch +common +com. +concurrent- +config +conscrypt-openjdk-uber +content-type +converter- +crate- +crt- +curvesapi +dd- +dec +derby +detector- +dnsjava +dom4j +drools- +dropbox- +duckdb +dynamodb +ecj +ejml- +encoder +endpoints- +error_ +eventstream +exporter- +failureaccess +fastutil +flatbuffers-java +flight- +flink- +flogger +flogger +fontbox +force- +gapic- +gax +gcsio +generex +google- +graal- +groovy +grpc- +gson +guava +guice +h2 +hadoop- +hamcrest +HdrHistogram +high- +hive- +hk2- +hop-action- +hop-assemblies- +hop-core +hop-databases- +hop-engine +hop-misc- +hop-resolvers- +hop-tech- +hop-transform- +!hop-transform-janino +hop-ui +hop-valuetypes- +hppc +hsqldb +httpclient +http-client- +httpcore +icu4j +ini4j +ipaddress +istack- +j2objc- +jackcess +jackcess +jackrabbit- +jackson- +jai-imageio-core +jakarta. +jamm +jandex +janino +java- +JavaEWAH +javafaker +java-libpst +javaparser-core +javapoet +javassist +javax. +jaxb- +jaxen +jbcrypt +jbig2-imageio +jcip-annotations +jcl-over-slf4j +jcodings +jcommander +jctools-core +jdbc-v2 +jdom2 +jediterm-core +jediterm-ui +jempbox +jersey- +jettison +jetty- +jffi +jhighlight +jline +jmatio +jna +jna- +joda-time +jollyday +joni +jsch +jsendnsca +json +json4s- +json- +jsoup +jsp-api +jspecify +jsr305 +js- +jt400 +jtokkit +jts-core +jul- +junit- +juniversalchardet +junrar +jvm- +jwarc +jython- +kafka- +kaml +kerb- +kie- +kinesis +kotlinpoet +kotlinpoet- +kotlin- +kotlinx- +kryo +langchain4j- +lang-tag +libphonenumber +lingua +listenablefuture +log4j- +logging- +logredactor +lombok +lz4- +managed-kafka-auth-login-handler +mariadb-java-client +metadata-extractor +metrics- +minimal-json +minio +minlog +mockito- +monetdb- +moshi +msal4j +mssql- +mvel2 +mxdump +mysql- +native-protocol +neo4j-java-driver +netty- +nimbus-jose-jwt +oauth2-oidc-sdk +objenesis- +odfdom-java +ohc- +okhttp +okio +openai4j +opencensus- +opencsv +opentelemetry- +opentest4j- +org.eclipse. +org.osgi. +osgi. +osgi- +paranamer +parquet- +parso +pdfbox +pdfbox- +perfmark-api +picocli +poi +postgresql +profiles +protobuf- +proto-google- +proton-j +psjava +pty4j +py4j +qpid-proton- +re2j +reactive-streams +reactor- +redshift-jdbc42 +regions +reload4j +reporter- +retrofit +rhino-all +RoaringBitmap +rome +rxjava +s2a- +s3 +saxon +scala- +sdk- +serializer +shared-resourcemapping +sigar +simple-xml-safe +sjk- +slf4j- +snakeyaml +snappy-java +snmp4j +snowball- +snowflake- +sns +spark- +SparseBitSet +splunk +spotbugs-annotations +sqlite-jdbc +sqs +sshd- +ST4 +stanford-corenlp +stax2-api +stream +sts +swagger- +swiftpoet +third-party-jackson- +threetenbp +threeten-extra +tika- +tinylog- +txw2 +ucanaccess +util +utils +vault-java-driver +vertica-jdbc +vorbis-java- +waffle-jna +webservices-api +websocket- +wire- +woodstox-core +wsdl4j +xalan +xbean- +xercesImpl +xml- +xmlbeans +xmlgraphics-commons +xmpbox +xmpcore +xom +xz +zstd-jni \ No newline at end of file diff --git a/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScannerTest.java b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScannerTest.java new file mode 100644 index 00000000000..926b195fcb4 --- /dev/null +++ b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/ClassLoaderScannerTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.Collection; +import org.apache.hop.pipeline.transforms.janino.function.JaninoFunction; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ClassLoaderScannerTest { + @Test + void findsAnnotatedMethodsInExplodedDirectory(@TempDir Path tempDir) throws Exception { + Path classesDir = tempDir.resolve("classes"); + FunctionGenerator.writeToDirectory(classesDir); + + try (URLClassLoader cl = + new URLClassLoader( + new URL[] {classesDir.toUri().toURL()}, + Thread.currentThread().getContextClassLoader())) { + ClassLoaderScanner scanner = new ClassLoaderScanner(); + Collection methods = + scanner.findMethodsWithAnnotationInPackage( + cl, "org.apache.hop.pipeline.transforms.janino.test", JaninoFunction.class); + + assertFalse(methods.isEmpty(), "Should find @JaninoFunction methods in exploded directory"); + assertTrue( + methods.stream().anyMatch(m -> "nvl".equals(m.getName())), "Should include nvl method"); + } + } + + @Test + void findsAnnotatedMethodsInJar(@TempDir Path tempDir) throws Exception { + Path jarFile = tempDir.resolve("test-functions.jar"); + FunctionGenerator.writeToJar(jarFile); + + try (URLClassLoader cl = + new URLClassLoader( + new URL[] {jarFile.toUri().toURL()}, Thread.currentThread().getContextClassLoader())) { + ClassLoaderScanner scanner = new ClassLoaderScanner(); + Collection methods = + scanner.findMethodsWithAnnotationInPackage( + cl, "org.apache.hop.pipeline.transforms.janino.test", JaninoFunction.class); + + assertFalse(methods.isEmpty(), "Should find @JaninoFunction methods in jar"); + assertTrue( + methods.stream().anyMatch(m -> "nvl".equals(m.getName())), "Should include nvl method"); + } + } + + @Test + void returnsEmptyForPackageWithNoMatchingAnnotation(@TempDir Path tempDir) throws Exception { + Path classesDir = tempDir.resolve("empty"); + FunctionGenerator.writeToDirectory(classesDir); + + try (URLClassLoader cl = + new URLClassLoader( + new URL[] {classesDir.toUri().toURL()}, + Thread.currentThread().getContextClassLoader())) { + ClassLoaderScanner scanner = new ClassLoaderScanner(); + Collection methods = + scanner.findMethodsWithAnnotationInPackage( + cl, "org.apache.hop.pipeline.transforms.janino.scanner", JaninoFunction.class); + + assertTrue(methods.isEmpty(), "Should find no @JaninoFunction when scanning wrong package"); + } + } + + @Test + void everyReturnedMethodCarriesTheAnnotation(@TempDir Path tempDir) throws Exception { + Path classesDir = tempDir.resolve("annotated"); + FunctionGenerator.writeToDirectory(classesDir); + + try (URLClassLoader cl = + new URLClassLoader( + new URL[] {classesDir.toUri().toURL()}, + Thread.currentThread().getContextClassLoader())) { + ClassLoaderScanner scanner = new ClassLoaderScanner(); + Collection methods = + scanner.findMethodsWithAnnotationInPackage( + cl, "org.apache.hop.pipeline.transforms.janino.test", JaninoFunction.class); + + assertFalse(methods.isEmpty()); + for (Method m : methods) { + assertNotNull( + m.getAnnotation(JaninoFunction.class), + "Method " + m + " must be annotated with @JaninoFunction"); + } + } + } +} diff --git a/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionGenerator.java b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionGenerator.java new file mode 100644 index 00000000000..c53c3a8b7c5 --- /dev/null +++ b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionGenerator.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import static org.apache.xbean.asm9.Opcodes.ACC_PUBLIC; +import static org.apache.xbean.asm9.Opcodes.ACC_STATIC; +import static org.apache.xbean.asm9.Opcodes.ACC_SUPER; +import static org.apache.xbean.asm9.Opcodes.ALOAD; +import static org.apache.xbean.asm9.Opcodes.ARETURN; +import static org.apache.xbean.asm9.Opcodes.IFNONNULL; +import static org.apache.xbean.asm9.Opcodes.V20; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import org.apache.xbean.asm9.AnnotationVisitor; +import org.apache.xbean.asm9.ClassWriter; +import org.apache.xbean.asm9.Label; +import org.apache.xbean.asm9.MethodVisitor; + +class FunctionGenerator { + + private static final String INTERNAL_CLASS_NAME = + "org/apache/hop/pipeline/transforms/janino/test/TestFunctions"; + + private static final String JANINO_FUNCTION_DESC = + "Lorg/apache/hop/pipeline/transforms/janino/function/JaninoFunction;"; + + static byte[] generateClass() { + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); + cw.visit(V20, ACC_PUBLIC | ACC_SUPER, INTERNAL_CLASS_NAME, null, "java/lang/Object", null); + + generateNvlMethod(cw); + + cw.visitEnd(); + return cw.toByteArray(); + } + + private static void generateNvlMethod(ClassWriter cw) { + MethodVisitor mv = + cw.visitMethod( + ACC_PUBLIC | ACC_STATIC, + "nvl", + "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", + null, + null); + + AnnotationVisitor av = mv.visitAnnotation(JANINO_FUNCTION_DESC, true); + av.visit("name", "nvl"); + av.visit("category", "General"); + av.visit("description", "Implements Oracle style NVL function."); + av.visit("syntax", "nvl(source, def)"); + av.visit("returns", "String"); + av.visit("semantics", "If source == null or empty return def"); + av.visit( + "examples", + "[{\"expression\":\"nvl(null,\\\"bar\\\")\",\"result\":\"bar\",\"level\":\"1\",\"comment\":\"null returns bar\"}]"); + av.visitEnd(); + + Label notNull = new Label(); + mv.visitCode(); + mv.visitVarInsn(ALOAD, 0); + mv.visitJumpInsn(IFNONNULL, notNull); + mv.visitVarInsn(ALOAD, 1); + mv.visitInsn(ARETURN); + mv.visitLabel(notNull); + mv.visitVarInsn(ALOAD, 0); + mv.visitInsn(ARETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + + static void writeToDirectory(Path dir) throws IOException { + Path classFile = dir.resolve(INTERNAL_CLASS_NAME + ".class"); + Files.createDirectories(classFile.getParent()); + Files.write(classFile, generateClass()); + } + + static void writeToJar(Path jarFile) throws IOException { + try (JarOutputStream jos = new JarOutputStream(Files.newOutputStream(jarFile))) { + String[] parts = INTERNAL_CLASS_NAME.split("/"); + String prefix = ""; + for (int i = 0; i < parts.length - 1; i++) { + prefix += parts[i] + "/"; + jos.putNextEntry(new JarEntry(prefix)); + jos.closeEntry(); + } + jos.putNextEntry(new JarEntry(INTERNAL_CLASS_NAME + ".class")); + jos.write(generateClass()); + jos.closeEntry(); + } + } +} diff --git a/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionLibTest.java b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionLibTest.java new file mode 100644 index 00000000000..a87ef713b42 --- /dev/null +++ b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/FunctionLibTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.List; +import org.apache.hop.core.plugins.IPlugin; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.plugins.TransformPluginType; +import org.apache.hop.pipeline.transforms.janino.function.FunctionDescription; +import org.apache.hop.pipeline.transforms.janino.function.FunctionLib; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; + +class FunctionLibTest { + // note: using mockito since current IoC design is not very friendly to tests + // with the static singleton pattern + @Test + void scansAndPopulatesFunctionsFromClasspath(@TempDir Path tempDir) throws Exception { + Path classesDir = tempDir.resolve("classes"); + FunctionGenerator.writeToDirectory(classesDir); + + try (URLClassLoader scanCl = + new URLClassLoader( + new URL[] {classesDir.toUri().toURL()}, FunctionLibTest.class.getClassLoader())) { + IPlugin plugin = mock(IPlugin.class); + PluginRegistry registry = mock(PluginRegistry.class); + when(registry.getPlugin(TransformPluginType.class, "Janino")).thenReturn(plugin); + when(registry.getClassLoader(plugin)).thenReturn(scanCl); + + try (MockedStatic mocked = mockStatic(PluginRegistry.class)) { + mocked.when(PluginRegistry::getInstance).thenReturn(registry); + + FunctionLib lib = new FunctionLib(); + List functions = lib.getFunctions(); + + assertFalse(functions.isEmpty(), "Should find at least one @JaninoFunction method"); + assertTrue( + functions.stream().anyMatch(f -> "nvl".equals(f.getName())), + "Should include the nvl method"); + + FunctionDescription nvl = + functions.stream().filter(f -> "nvl".equals(f.getName())).findFirst().get(); + assertEquals("General", nvl.getCategory()); + assertEquals("String", nvl.getReturns()); + assertNotNull(nvl.getFunctionExamples()); + assertFalse(nvl.getFunctionExamples().isEmpty()); + } + } + } +} diff --git a/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoaderTest.java b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoaderTest.java new file mode 100644 index 00000000000..3abb61bd106 --- /dev/null +++ b/plugins/transforms/janino/src/test/java/org/apache/hop/pipeline/transforms/janino/scanner/JarExclusionsLoaderTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.janino.scanner; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.function.Predicate; +import org.junit.jupiter.api.Test; + +class JarExclusionsLoaderTest { + @Test + void loadsFromResourceFile() { + var factory = new JarExclusionsLoader(); + Predicate keep = factory.load("ClassLoaderScanner.ignored-jars.txt"); + assertFalse(keep.test("asm-9.8.jar")); + assertFalse(keep.test("hop-transform-rowgenerator-2.19.0.jar")); + assertTrue(keep.test("hop-transform-janino-2.19.0.jar")); + } + + @Test + void excludesKnownPrefix() { + var keep = exclusions("asm"); + assertFalse(keep.test("asm-9.8.jar")); + assertFalse(keep.test("asm")); + assertTrue(keep.test("jackson-core-2.21.1.jar")); + } + + @Test + void notIgnoredPrefixTakesPrecedence() { + var keep = exclusions("hop-transform-\n!hop-transform-janino"); + assertFalse(keep.test("hop-transform-rowgenerator-2.19.0.jar")); + assertTrue(keep.test("hop-transform-janino-2.19.0.jar")); + } + + @Test + void emptyContentKeepsEverything() { + var keep = exclusions(""); + assertTrue(keep.test("anything.jar")); + } + + @Test + void skipsBlankLines() { + var keep = exclusions("\n\nasm\n\n"); + assertFalse(keep.test("asm-9.8.jar")); + assertTrue(keep.test("jackson-core.jar")); + } + + @Test + void multipleIgnoresAndNotIgnores() { + var keep = exclusions("jackson-\n!jackson-databind\nasm\n!asm-analysis"); + assertFalse(keep.test("jackson-core-2.21.1.jar")); + assertTrue(keep.test("jackson-databind-2.21.1.jar")); + assertFalse(keep.test("asm-9.8.jar")); + assertTrue(keep.test("asm-analysis-9.8.jar")); + } + + private Predicate exclusions(String content) { + return new JarExclusionsLoader() + .load(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); + } +}