Rework janino scanner to make it lighter in mem and faster

core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java

@@ -21,10 +21,13 @@
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.security.ProtectionDomain;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
 
 public class HopURLClassLoader extends URLClassLoader {
 
   private String name;
+  private final ConcurrentHashMap<Class<?>, Object> cache = new ConcurrentHashMap<>();
 
   public HopURLClassLoader(URL[] url, ClassLoader classLoader) {
     super(url, classLoader);
@@ -35,9 +38,31 @@ public HopURLClassLoader(URL[] url, ClassLoader classLoader, String name) {
     this.name = name;
   }
 
+  public <T> T get(final Class<T> key) {
+    return key.cast(cache.get(key));
+  }
+
+  public <T> T computeIfAbsent(final Class<T> key, Supplier<T> provider) {
+    return key.cast(cache.computeIfAbsent(key, k -> provider));
+  }
+
   @Override
   protected void addURL(URL url) {
     super.addURL(url);
+    // invalidate the cache since it is wrong if related to the classloader,
+    // do not lock since we assume addURL is called in a safe context
+    cache.values().stream()
+        .filter(AutoCloseable.class::isInstance)
+        .map(AutoCloseable.class::cast)
+        .forEach(
+            it -> {
+              try {
+                it.close();
+              } catch (final Exception e) {
+                // no-op
+              }
+            });
+    cache.clear();
   }
 
   @Override

core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java

@@ -17,14 +17,17 @@
 package org.apache.hop.core.vfs;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import com.sun.net.httpserver.HttpServer;
 import com.sun.net.httpserver.HttpsConfigurator;
 import com.sun.net.httpserver.HttpsServer;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -70,6 +73,7 @@ class HopVfsNetworkProvidersTest {
   private static final String FTP_PASS = "secret";
   private static final String SFTP_USER = "alice";
   private static final String SFTP_PASS = "secret";
+  private static String LOCALHOST;
 
   @TempDir static Path sharedRoot;
 
@@ -93,19 +97,27 @@ class HopVfsNetworkProvidersTest {
   private static SshServer sshServer;
   private static int sftpPort;
 
+  static {
+    try {
+      LOCALHOST = InetAddress.getLocalHost().getHostAddress();
+    } catch (final UnknownHostException e) {
+      fail(e);
+    }
+  }
+
   @BeforeAll
   static void startServers() throws Exception {
     keyStorePath = generateTestKeyStore();
 
     // HTTP
-    httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+    httpServer = HttpServer.create(new InetSocketAddress(LOCALHOST, 0), 0);
     httpPort = httpServer.getAddress().getPort();
     httpServer.createContext("/payload.txt", new FixedPayloadHandler("http-payload"));
     httpServer.start();
 
     // HTTPS
     SSLContext sslContext = buildServerSslContext(keyStorePath);
-    httpsServer = HttpsServer.create(new InetSocketAddress("localhost", 0), 0);
+    httpsServer = HttpsServer.create(new InetSocketAddress(LOCALHOST, 0), 0);
     httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
     httpsServer.createContext("/secure.txt", new FixedPayloadHandler("https-payload"));
     httpsServer.start();
@@ -158,19 +170,22 @@ static void stopServers() throws Exception {
   @Test
   @DisplayName("http:// fetches a payload from an embedded HttpServer")
   void httpProviderReadsFromEmbeddedServer() throws Exception {
-    assertEquals("http-payload", readToString("http://localhost:" + httpPort + "/payload.txt"));
+    assertEquals(
+        "http-payload", readToString("http://" + LOCALHOST + ":" + httpPort + "/payload.txt"));
   }
 
   @Test
   @DisplayName("https:// fetches a payload over TLS from an embedded HttpsServer")
   void httpsProviderReadsFromEmbeddedServer() throws Exception {
-    assertEquals("https-payload", readToString("https://localhost:" + httpsPort + "/secure.txt"));
+    assertEquals(
+        "https-payload", readToString("https://" + LOCALHOST + ":" + httpsPort + "/secure.txt"));
   }
 
   @Test
   @DisplayName("ftp:// fetches a payload from an embedded Apache FtpServer")
   void ftpProviderReadsFromEmbeddedServer() throws Exception {
-    String url = "ftp://" + FTP_USER + ":" + FTP_PASS + "@localhost:" + ftpPort + "/greeting.txt";
+    String url =
+        "ftp://" + FTP_USER + ":" + FTP_PASS + "@" + LOCALHOST + ":" + ftpPort + "/greeting.txt";
     FileSystemOptions opts = new FileSystemOptions();
     FtpFileSystemConfigBuilder.getInstance().setPassiveMode(opts, true);
     assertEquals("ftp-payload", readWithOptions(url, opts));
@@ -179,7 +194,8 @@ void ftpProviderReadsFromEmbeddedServer() throws Exception {
   @Test
   @DisplayName("ftps:// fetches a payload over TLS from an embedded Apache FtpServer")
   void ftpsProviderReadsFromEmbeddedServer() throws Exception {
-    String url = "ftps://" + FTP_USER + ":" + FTP_PASS + "@localhost:" + ftpsPort + "/greeting.txt";
+    String url =
+        "ftps://" + FTP_USER + ":" + FTP_PASS + "@" + LOCALHOST + ":" + ftpsPort + "/greeting.txt";
     FileSystemOptions opts = new FileSystemOptions();
     FtpsFileSystemConfigBuilder ftps = FtpsFileSystemConfigBuilder.getInstance();
     ftps.setPassiveMode(opts, true);
@@ -191,7 +207,15 @@ void ftpsProviderReadsFromEmbeddedServer() throws Exception {
   @DisplayName("sftp:// fetches a payload from an embedded Apache MINA SSHD server")
   void sftpProviderReadsFromEmbeddedServer() throws Exception {
     String url =
-        "sftp://" + SFTP_USER + ":" + SFTP_PASS + "@localhost:" + sftpPort + "/greeting.txt";
+        "sftp://"
+            + SFTP_USER
+            + ":"
+            + SFTP_PASS
+            + "@"
+            + LOCALHOST
+            + ":"
+            + sftpPort
+            + "/greeting.txt";
     assertEquals("sftp-payload", readToString(url));
   }
 
@@ -238,7 +262,7 @@ private static Path generateTestKeyStore() throws Exception {
                 "-dname",
                 "CN=localhost, OU=Hop, O=Apache, L=Test, S=Test, C=US",
                 "-ext",
-                "SAN=DNS:localhost,IP:127.0.0.1",
+                "SAN=DNS:localhost,IP:" + LOCALHOST,
                 "-noprompt")
             .redirectErrorStream(true)
             .start();
@@ -302,7 +326,7 @@ private static FtpServerStart startFtp(Path home, boolean tls) throws Exception
 
   private static SshServer startSftp(Path home) throws IOException {
     SshServer sshd = SshServer.setUpDefaultServer();
-    sshd.setHost("localhost");
+    sshd.setHost(LOCALHOST);
     sshd.setPort(0);
     sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(sharedRoot.resolve("hostkey.ser")));
     sshd.setPasswordAuthenticator(AcceptAllPasswordAuthenticator.INSTANCE);

plugins/transforms/janino/pom.xml

@@ -39,6 +39,11 @@
             <artifactId>hop-transform-rowgenerator</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.xbean</groupId>
+            <artifactId>xbean-finder-shaded</artifactId>
+            <version>4.30</version>
+        </dependency>
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>

plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/function/FunctionLib.java

@@ -31,13 +31,14 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.plugins.HopURLClassLoader;
 import org.apache.hop.core.plugins.IPlugin;
 import org.apache.hop.core.plugins.PluginRegistry;
 import org.apache.hop.core.plugins.TransformPluginType;
 import org.apache.hop.core.util.Utils;
+import org.apache.hop.pipeline.transforms.janino.scanner.ClassLoaderScanner;
 
 public class FunctionLib {
-
   private List<FunctionDescription> functions;
 
   public FunctionLib() throws HopException {
@@ -46,49 +47,56 @@ public FunctionLib() throws HopException {
       PluginRegistry registry = PluginRegistry.getInstance();
       IPlugin plugin = registry.getPlugin(TransformPluginType.class, "Janino");
       ClassLoader loader = registry.getClassLoader(plugin);
-      Set<Class<?>> classes =
-          findAllClassesUsingGoogleGuice(loader, "org.apache.hop.pipeline.transforms.janino");
-
-      for (Class<?> clazz : classes) {
-        Method[] methods = clazz.getMethods();
-        for (Method method : methods) {
-          JaninoFunction annotation = method.getAnnotation(JaninoFunction.class);
-          List<FunctionExample> functionExamples = new ArrayList<>();
-          if (annotation != null) {
-            if (!Utils.isEmpty(annotation.examples())) {
-              ObjectMapper mapper = new ObjectMapper();
-              JsonNode arrayNode = mapper.readTree(annotation.examples());
-              for (JsonNode jsonNode : arrayNode) {
-                functionExamples.add(
-                    new FunctionExample(
-                        jsonNode.get("expression").asText(),
-                        jsonNode.get("result").asText(),
-                        jsonNode.get("level").asText(),
-                        jsonNode.get("comment").asText()));
-              }
-            }
-
-            FunctionDescription functionDescription =
-                new FunctionDescription(
-                    annotation.category(),
-                    annotation.name(),
-                    annotation.description(),
-                    annotation.syntax(),
-                    annotation.returns(),
-                    null,
-                    annotation.semantics(),
-                    clazz.getCanonicalName(),
-                    functionExamples);
-            functions.add(functionDescription);
-          }
+
+      if (loader instanceof HopURLClassLoader hucl) {
+        var cached = hucl.get(CachedFunctions.class);
+        if (cached != null) {
+          functions.addAll(cached.functions());
+          return;
         }
       }
 
+      doScan(loader);
     } catch (Exception e) {
       throw new HopException(e);
     }
   }
 
+  private void doScan(ClassLoader loader) throws IOException {
+    for (Method method :
+        new ClassLoaderScanner()
+            .findMethodsWithAnnotationInPackage(
+                loader, "org.apache.hop.pipeline.transforms.janino", JaninoFunction.class)) {
+      JaninoFunction annotation = method.getAnnotation(JaninoFunction.class);
+      List<FunctionExample> functionExamples = new ArrayList<>();
+      if (!Utils.isEmpty(annotation.examples())) {
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode arrayNode = mapper.readTree(annotation.examples());
+        for (JsonNode jsonNode : arrayNode) {
+          functionExamples.add(
+              new FunctionExample(
+                  jsonNode.get("expression").asText(),
+                  jsonNode.get("result").asText(),
+                  jsonNode.get("level").asText(),
+                  jsonNode.get("comment").asText()));
+        }
+      }
+
+      FunctionDescription functionDescription =
+          new FunctionDescription(
+              annotation.category(),
+              annotation.name(),
+              annotation.description(),
+              annotation.syntax(),
+              annotation.returns(),
+              null,
+              annotation.semantics(),
+              method.getDeclaringClass().getCanonicalName(),
+              functionExamples);
+      functions.add(functionDescription);
+    }
+  }
+
   /**
    * @return the functions
    */
@@ -172,6 +180,7 @@ public FunctionDescription getFunctionDescription(String functionName) {
     return null;
   }
 
+  @Deprecated // shouldn't be public, kept for legacy and external usage
   public Set<Class<?>> findAllClassesUsingGoogleGuice(ClassLoader classLoader, String packageName)
       throws IOException {
     return ClassPath.from(classLoader).getAllClasses().stream()
@@ -188,4 +197,6 @@ public Set<Class<?>> findAllClassesUsingGoogleGuice(ClassLoader classLoader, Str
             })
         .collect(Collectors.toSet());
   }
+
+  private record CachedFunctions(List<FunctionDescription> functions) {}
 }