diff --git a/build.sc b/build.sc index de1d623d6f09..d66f96f449dd 100755 --- a/build.sc +++ b/build.sc @@ -309,6 +309,7 @@ object main extends MillModule { s""" |package mill | + |/** Generated by mill. */ |object BuildInfo { | /** Scala version used to compile mill core. */ | val scalaVersion = "$scalaVersion" @@ -341,6 +342,21 @@ object main extends MillModule { override def ivyDeps = Agg( Deps.ipcsocketExcludingJna ) + def generatedBuildInfo = T{ + val dest = T.dest + val code = + s"""package mill.main.client; + | + |/** Generated by mill. */ + |public class BuildInfo { + | /** Mill version. */ + | public static String millVersion() { return "${millVersion()}"; } + |} + |""".stripMargin + os.write(dest / "mill" / "main" / "client" / "BuildInfo.java", code, createFolders = true) + Seq(PathRef(dest)) + } + override def generatedSources: T[Seq[PathRef]] = super.generatedSources() ++ generatedBuildInfo() object test extends Tests with TestModule.Junit4 { override def ivyDeps = Agg(Deps.junitInterface, Deps.lambdaTest) } @@ -841,7 +857,7 @@ def launcherScript( shellClassPath: Agg[String], cmdClassPath: Agg[String] ) = { - val millMainClass = "mill.MillMain" + val millMainClass = "mill.main.client.MillClientMain" val millClientMainClass = "mill.main.client.MillClientMain" mill.modules.Jvm.universalScript( @@ -875,6 +891,7 @@ def launcherScript( | "-X"*) mill_jvm_opts="$${mill_jvm_opts} $$line" | esac | done <"$$mill_jvm_opts_file" + | mill_jvm_opts="$${mill_jvm_opts} -Dmill.jvm_opts_applied=true" | fi |} | @@ -1023,7 +1040,7 @@ object dev extends MillModule { def run(args: String*) = T.command { args match { - case Nil => mill.eval.Result.Failure("Need to pass in cwd as first argument to dev.run") + case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") case wd0 +: rest => val wd = os.Path(wd0, os.pwd) os.makeDir.all(wd) @@ -1032,7 +1049,7 @@ object dev extends MillModule { forkEnv(), workingDir = wd ) - mill.eval.Result.Success(()) + mill.api.Result.Success(()) } } diff --git a/main/client/src/mill/main/client/IsolatedMillMainLoader.java b/main/client/src/mill/main/client/IsolatedMillMainLoader.java new file mode 100644 index 000000000000..d7ebacc4406a --- /dev/null +++ b/main/client/src/mill/main/client/IsolatedMillMainLoader.java @@ -0,0 +1,79 @@ +package mill.main.client; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +class IsolatedMillMainLoader { + + public static class LoadResult { + + public final Optional millMainMethod; + public final boolean canLoad; + public final long loadTime; + + public LoadResult(Optional millMainMethod, final long loadTime) { + this.millMainMethod = millMainMethod; + this.canLoad = millMainMethod.isPresent(); + this.loadTime = loadTime; + } + } + + private static Optional canLoad = Optional.empty(); + + public static LoadResult load() { + if (canLoad.isPresent()) { + return canLoad.get(); + } else { + long startTime = System.currentTimeMillis(); + Optional millMainMethod = Optional.empty(); + try { + Class millMainClass = IsolatedMillMainLoader.class.getClassLoader().loadClass("mill.MillMain"); + Method mainMethod = millMainClass.getMethod("main", String[].class); + millMainMethod = Optional.of(mainMethod); + } catch (ClassNotFoundException | NoSuchMethodException e) { + millMainMethod = Optional.empty(); + } + + long loadTime = System.currentTimeMillis() - startTime; + LoadResult result = new LoadResult(millMainMethod, loadTime); + canLoad = Optional.of(result); + return result; + } + } + + public static void runMain(String[] args) throws Exception { + LoadResult loadResult = load(); + if (loadResult.millMainMethod.isPresent()) { + if (!MillEnv.millJvmOptsAlreadyApplied() && MillEnv.millJvmOptsFile().exists()) { + System.err.println("Launching Mill as sub-process ..."); + int exitVal = launchMillAsSubProcess(args); + System.exit(exitVal); + } else { + // launch mill in-process + // it will call System.exit for us + Method mainMethod = loadResult.millMainMethod.get(); + mainMethod.invoke(null, new Object[]{args}); + } + } else { + throw new RuntimeException("Cannot load mill.MillMain class"); + } + } + + private static int launchMillAsSubProcess(String[] args) throws Exception { + boolean setJnaNoSys = System.getProperty("jna.nosys") == null; + + List l = new ArrayList<>(); + l.addAll(MillEnv.millLaunchJvmCommand(setJnaNoSys)); + l.add("mill.MillMain"); + l.addAll(Arrays.asList(args)); + + Process running = new ProcessBuilder() + .command(l) + .inheritIO() + .start(); + return running.waitFor(); + } +} diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index 8f2bc24e164e..a1912a547fb4 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -11,83 +11,41 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; import java.net.Socket; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.lang.Math; import java.util.ArrayList; -import java.util.Base64; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Scanner; +/** + * This is a Java implementation to speed up repetitive starts. + * A Scala implementation would result in the JVM loading much more classes almost doubling the start-up times. + */ public class MillClientMain { // use methods instead of constants to avoid inlining by compiler - public static final int ExitClientCodeCannotReadFromExitCodeFile() { return 1; } - public static final int ExitServerCodeWhenIdle() { return 0; } - public static final int ExitServerCodeWhenVersionMismatch() { return 101; } - - static void initServer(String lockBase, boolean setJnaNoSys) throws IOException, URISyntaxException { - - String selfJars = ""; - List vmOptions = new ArrayList<>(); - String millOptionsPath = System.getProperty("MILL_OPTIONS_PATH"); - if (millOptionsPath != null) { - // read MILL_CLASSPATH from file MILL_OPTIONS_PATH - Properties millProps = new Properties(); - millProps.load(new FileInputStream(millOptionsPath)); - for(final String k: millProps.stringPropertyNames()){ - String propValue = millProps.getProperty(k); - if ("MILL_CLASSPATH".equals(k)){ - selfJars = propValue; - } - } - } else { - // read MILL_CLASSPATH from file sys props - selfJars = System.getProperty("MILL_CLASSPATH"); - } - - final Properties sysProps = System.getProperties(); - for (final String k: sysProps.stringPropertyNames()){ - if (k.startsWith("MILL_") && !"MILL_CLASSPATH".equals(k)) { - vmOptions.add("-D" + k + "=" + sysProps.getProperty(k)); - } - } - if (selfJars == null || selfJars.trim().isEmpty()) { - throw new RuntimeException("MILL_CLASSPATH is empty!"); - } - if (setJnaNoSys) { - vmOptions.add("-Djna.nosys=true"); - } + public static final int ExitClientCodeCannotReadFromExitCodeFile() { + return 1; + } - String millJvmOptsPath = System.getProperty("MILL_JVM_OPTS_PATH"); - if (millJvmOptsPath == null) { - millJvmOptsPath = ".mill-jvm-opts"; - } + public static final int ExitServerCodeWhenIdle() { + return 0; + } - File millJvmOptsFile = new File(millJvmOptsPath); - if (millJvmOptsFile.exists()) { - try (Scanner sc = new Scanner(millJvmOptsFile)) { - while (sc.hasNextLine()) { - String arg = sc.nextLine(); - vmOptions.add(arg); - } - } - } + public static final int ExitServerCodeWhenVersionMismatch() { + return 101; + } + static void initServer(String lockBase, boolean setJnaNoSys) throws IOException, URISyntaxException { List l = new ArrayList<>(); - l.add(System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"); - l.addAll(vmOptions); - l.add("-cp"); - l.add(String.join(File.pathSeparator, selfJars.split(","))); + l.addAll(MillEnv.millLaunchJvmCommand(setJnaNoSys)); l.add("mill.main.MillServerMain"); l.add(lockBase); @@ -95,27 +53,43 @@ static void initServer(String lockBase, boolean setJnaNoSys) throws IOException, File stderr = new java.io.File(lockBase + "/stderr"); new ProcessBuilder() - .command(l) - .redirectOutput(stdout) - .redirectError(stderr) - .start(); + .command(l) + .redirectOutput(stdout) + .redirectError(stderr) + .start(); } - private static String sha1HashPath(String path) throws NoSuchAlgorithmException, UnsupportedEncodingException { - MessageDigest md = MessageDigest.getInstance("SHA1"); - md.reset(); - byte[] pathBytes = path.getBytes("UTF-8"); - md.update(pathBytes); - byte[] digest = md.digest(); - return Base64.getEncoder().encodeToString(digest); - } + public static void main(String[] args) throws Exception { + if (args.length > 1) { + String firstArg = args[0]; + if (Arrays.asList("-i", "--interactive", "--no-server", "--repl", "--bsp").contains(firstArg)) { + // start in no-server mode + IsolatedMillMainLoader.runMain(args); + return; + } + } - public static void main(String[] args) throws Exception{ - int exitCode = main0(args); - if (exitCode == ExitServerCodeWhenVersionMismatch()) { - exitCode = main0(args); + // start in client-server mode + try { + int exitCode = main0(args); + if (exitCode == ExitServerCodeWhenVersionMismatch()) { + exitCode = main0(args); + } + System.exit(exitCode); + } catch (MillServerCouldNotBeStarted e) { + // TODO: try to run in-process + System.err.println("Could not start a Mill server process.\n" + + "This could be caused by too many already running Mill instances " + + "or by an unsupported platform.\n"); + if (IsolatedMillMainLoader.load().canLoad) { + System.err.println("Trying to run Mill in-process ..."); + IsolatedMillMainLoader.runMain(args); + } else { + System.err.println("Loading Mill in-process isn't possible.\n" + + "Please check your Mill installation!"); + throw e; + } } - System.exit(exitCode); } public static int main0(String[] args) throws Exception { @@ -124,8 +98,8 @@ public static int main0(String[] args) throws Exception { if (setJnaNoSys) { System.setProperty("jna.nosys", "true"); } - - String jvmHomeEncoding = sha1HashPath(System.getProperty("java.home")); + + String jvmHomeEncoding = Util.sha1Hash(System.getProperty("java.home")); int serverProcessesLimit = getServerProcessesLimit(jvmHomeEncoding); int index = 0; @@ -173,21 +147,28 @@ public static int main0(String[] args) throws Exception { } } } - throw new Exception("Reached max server processes limit: " + serverProcessesLimit); + throw new MillServerCouldNotBeStarted("Reached max server processes limit: " + serverProcessesLimit); } - public static int run(String lockBase, - Runnable initServer, - Locks locks, - InputStream stdin, - OutputStream stdout, - OutputStream stderr, - String[] args, - Map env) throws Exception{ + public static class MillServerCouldNotBeStarted extends Exception { + public MillServerCouldNotBeStarted(String msg) { + super(msg); + } + } + + public static int run( + String lockBase, + Runnable initServer, + Locks locks, + InputStream stdin, + OutputStream stdout, + OutputStream stderr, + String[] args, + Map env) throws Exception { try (FileOutputStream f = new FileOutputStream(lockBase + "/run")) { f.write(System.console() != null ? 1 : 0); - Util.writeString(f, System.getProperty("MILL_VERSION")); + Util.writeString(f, BuildInfo.millVersion()); Util.writeArgs(args, f); Util.writeMap(env, f); } @@ -209,11 +190,11 @@ public static int run(String lockBase, while (ioSocket == null && System.currentTimeMillis() - retryStart < 5000) { try { - String socketBaseName = "mill-" + md5hex(new File(lockBase).getCanonicalPath()); - ioSocket = Util.isWindows? - new Win32NamedPipeSocket(Util.WIN32_PIPE_PREFIX + socketBaseName) - : new UnixDomainSocket(lockBase + "/" + socketBaseName + "-io"); - } catch (Throwable e){ + String socketBaseName = "mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()); + ioSocket = Util.isWindows ? + new Win32NamedPipeSocket(Util.WIN32_PIPE_PREFIX + socketBaseName) + : new UnixDomainSocket(lockBase + "/" + socketBaseName + "-io"); + } catch (Throwable e) { socketThrowable = e; Thread.sleep(1); } @@ -247,8 +228,8 @@ public static int run(String lockBase, // 5 processes max private static int getServerProcessesLimit(String jvmHomeEncoding) { File outFolder = new File("out"); - String[] totalProcesses = outFolder.list((dir,name) -> name.startsWith("mill-worker-")); - String[] thisJdkProcesses = outFolder.list((dir,name) -> name.startsWith("mill-worker-" + jvmHomeEncoding)); + String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-")); + String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-" + jvmHomeEncoding)); int processLimit = 5; if (totalProcesses != null) { @@ -261,12 +242,11 @@ private static int getServerProcessesLimit(String jvmHomeEncoding) { return processLimit; } - /** @return Hex encoded MD5 hash of input string. */ + /** + * @deprecated Use {@link Util#md5hex(String)} instead. (Deprecated since after Mill 0.10.0) + */ public static String md5hex(String str) throws NoSuchAlgorithmException { - return hexArray(MessageDigest.getInstance("md5").digest(str.getBytes(StandardCharsets.UTF_8))); + return Util.md5hex(str); } - private static String hexArray(byte[] arr) { - return String.format("%0" + (arr.length << 1) + "x", new BigInteger(1, arr)); - } } diff --git a/main/client/src/mill/main/client/MillEnv.java b/main/client/src/mill/main/client/MillEnv.java new file mode 100644 index 000000000000..c96589bf21d1 --- /dev/null +++ b/main/client/src/mill/main/client/MillEnv.java @@ -0,0 +1,127 @@ +package mill.main.client; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.*; + +public class MillEnv { + + static File millJvmOptsFile() { + String millJvmOptsPath = System.getenv("MILL_JVM_OPTS_PATH"); + if (millJvmOptsPath == null || millJvmOptsPath.trim().equals("")) { + millJvmOptsPath = ".mill-jvm-opts"; + } + return new File(millJvmOptsPath).getAbsoluteFile(); + } + + static boolean millJvmOptsAlreadyApplied() { + final String propAppliedProp = System.getProperty("mill.jvm_opts_applied"); + return propAppliedProp != null && propAppliedProp.equals("true"); + } + + static boolean isWin() { + return System.getProperty("os.name", "").startsWith("Windows"); + } + + static String javaExe() { + final String javaHome = System.getProperty("java.home"); + if (javaHome != null && !javaHome.isEmpty()) { + final File exePath = new File( + javaHome + File.separator + + "bin" + File.separator + + "java" + (isWin() ? ".exe" : "") + ); + if (exePath.exists()) { + return exePath.getAbsolutePath(); + } + } + return "java"; + } + + static String[] millClasspath() { + String selfJars = ""; + List vmOptions = new LinkedList<>(); + String millOptionsPath = System.getProperty("MILL_OPTIONS_PATH"); + if (millOptionsPath != null) { + + // read MILL_CLASSPATH from file MILL_OPTIONS_PATH + Properties millProps = new Properties(); + try (FileInputStream is = new FileInputStream(millOptionsPath)) { + millProps.load(is); + } catch (IOException e) { + throw new RuntimeException("Could not load '" + millOptionsPath + "'", e); + } + + for (final String k : millProps.stringPropertyNames()) { + String propValue = millProps.getProperty(k); + if ("MILL_CLASSPATH".equals(k)) { + selfJars = propValue; + } + } + } else { + // read MILL_CLASSPATH from file sys props + selfJars = System.getProperty("MILL_CLASSPATH"); + } + + if (selfJars == null || selfJars.trim().isEmpty()) { + // We try to use the currently local classpath as MILL_CLASSPATH + selfJars = System.getProperty("java.class.path").replace(File.pathSeparator, ","); + } + + if (selfJars == null || selfJars.trim().isEmpty()) { + throw new RuntimeException("MILL_CLASSPATH is empty!"); + } + return selfJars.split("[,]"); + } + + static List millLaunchJvmCommand(boolean setJnaNoSys) { + final List vmOptions = new ArrayList<>(); + + // Java executable + vmOptions.add(javaExe()); + + // jna + if (setJnaNoSys) { + vmOptions.add("-Djna.nosys=true"); + } + + // sys props + final Properties sysProps = System.getProperties(); + for (final String k : sysProps.stringPropertyNames()) { + if (k.startsWith("MILL_") && !"MILL_CLASSPATH".equals(k)) { + vmOptions.add("-D" + k + "=" + sysProps.getProperty(k)); + } + } + + // extra opts + File millJvmOptsFile = millJvmOptsFile(); + if (millJvmOptsFile.exists()) { + vmOptions.addAll(readMillJvmOpts()); + } + + vmOptions.add("-cp"); + vmOptions.add(String.join(File.pathSeparator, millClasspath())); + + return vmOptions; + } + + static List readMillJvmOpts() { + final List vmOptions = new LinkedList<>(); + try ( + final Scanner sc = new Scanner(millJvmOptsFile()) + ) { + while (sc.hasNextLine()) { + String arg = sc.nextLine(); + if (!arg.trim().isEmpty() && arg.startsWith("#")) { + vmOptions.add(arg); + } + } + } catch (FileNotFoundException e) { + // ignored + } + return vmOptions; + } + +} diff --git a/main/client/src/mill/main/client/Util.java b/main/client/src/mill/main/client/Util.java index da74eefe6d5e..d82524538e2b 100644 --- a/main/client/src/mill/main/client/Util.java +++ b/main/client/src/mill/main/client/Util.java @@ -3,7 +3,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -26,10 +32,11 @@ public static String[] parseArgs(InputStream argStream) throws IOException { } return args; } + public static void writeArgs(String[] args, OutputStream argStream) throws IOException { writeInt(argStream, args.length); - for(String arg: args){ + for (String arg : args) { writeString(argStream, arg); } } @@ -63,10 +70,10 @@ public static String readString(InputStream inputStream) throws IOException { final int length = readInt(inputStream); final byte[] arr = new byte[length]; int total = 0; - while(total < length){ - int res = inputStream.read(arr, total, length-total); + while (total < length) { + int res = inputStream.read(arr, total, length - total); if (res == -1) throw new IOException("Incomplete String"); - else{ + else { total += res; } } @@ -79,17 +86,38 @@ public static void writeString(OutputStream outputStream, String string) throws outputStream.write(bytes); } - public static void writeInt(OutputStream out, int i) throws IOException{ - out.write((byte)(i >>> 24)); - out.write((byte)(i >>> 16)); - out.write((byte)(i >>> 8)); - out.write((byte)i); + public static void writeInt(OutputStream out, int i) throws IOException { + out.write((byte) (i >>> 24)); + out.write((byte) (i >>> 16)); + out.write((byte) (i >>> 8)); + out.write((byte) i); } - public static int readInt(InputStream in) throws IOException{ + + public static int readInt(InputStream in) throws IOException { return ((in.read() & 0xFF) << 24) + - ((in.read() & 0xFF) << 16) + - ((in.read() & 0xFF) << 8) + - (in.read() & 0xFF); + ((in.read() & 0xFF) << 16) + + ((in.read() & 0xFF) << 8) + + (in.read() & 0xFF); + } + + /** + * @return Hex encoded MD5 hash of input string. + */ + public static String md5hex(String str) throws NoSuchAlgorithmException { + return hexArray(MessageDigest.getInstance("md5").digest(str.getBytes(StandardCharsets.UTF_8))); + } + + private static String hexArray(byte[] arr) { + return String.format("%0" + (arr.length << 1) + "x", new BigInteger(1, arr)); + } + + static String sha1Hash(String path) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("SHA1"); + md.reset(); + byte[] pathBytes = path.getBytes(StandardCharsets.UTF_8); + md.update(pathBytes); + byte[] digest = md.digest(); + return Base64.getEncoder().encodeToString(digest); } } diff --git a/main/src/mill/main/MillServerMain.scala b/main/src/mill/main/MillServerMain.scala index ba7452fa18a1..852b904feae4 100644 --- a/main/src/mill/main/MillServerMain.scala +++ b/main/src/mill/main/MillServerMain.scala @@ -96,7 +96,7 @@ class Server[T]( var running = true while (running) { Server.lockBlock(locks.serverLock) { - val socketBaseName = "mill-" + MillClientMain.md5hex(new File(lockBase).getCanonicalPath) + val socketBaseName = "mill-" + Util.md5hex(new File(lockBase).getCanonicalPath) val (serverSocket, socketClose) = if (Util.isWindows) { val socketName = Util.WIN32_PIPE_PREFIX + socketBaseName