diff --git a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java index 10beee6..1ad5f22 100644 --- a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java +++ b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java @@ -41,7 +41,25 @@ public class TerraformClient implements AutoCloseable { // spend". They happen to agree today. private static final long CLOSE_GRACE_PERIOD_SECONDS = 10; - private final ExecutorService executor = Executors.newWorkStealingPool(); + private static final AtomicInteger CLIENT_THREAD_COUNT = new AtomicInteger(); + + // A cached pool, not newWorkStealingPool(). Every task submitted here is + // blocking I/O - two pipe readers and a waitFor() per command - and a + // ForkJoinPool sized to availableProcessors() only grows workers for a + // ManagedBlocker, which neither readLine() nor waitFor() is. So N concurrent + // commands need 3N blocked threads and can never have more than + // availableProcessors(): measured on 4 cores, the second concurrent command + // already has a starved reader, and once that reader's pipe buffer fills the + // child itself blocks on write and stops making progress. A cached pool + // grows to the blocking work and reaps idle threads after 60s. + private final ExecutorService executor = Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "terraform-client-" + CLIENT_THREAD_COUNT.incrementAndGet()); + // Daemon, as ForkJoinPool's workers were, so the library never holds JVM + // exit open. Pinned classloader for the same reason as TERMINATOR's. + thread.setDaemon(true); + thread.setContextClassLoader(TerraformClient.class.getClassLoader()); + return thread; + }); // Final with an initializer and no accessors on purpose: that is what keeps // these two off the generated builder, off the all-args constructor and out // of the published API. A plain volatile boolean for the flag gave Lombok a diff --git a/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java b/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java index d9fd334..00255d1 100644 --- a/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java +++ b/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; @@ -30,7 +31,20 @@ @DisabledOnOs(OS.WINDOWS) class ProcessLauncherTest { - private static final ExecutorService POOL = Executors.newWorkStealingPool(); + private static final AtomicInteger POOL_THREADS = new AtomicInteger(); + + // Cached, not work-stealing, for the same reason the client's own pool is: + // every task here is blocking I/O, and a ForkJoinPool sized to + // availableProcessors() cannot run more than that many blocked tasks, so + // errorStreamIsFullyCapturedWhenExitCodeFutureCompletes below deadlocks at + // one core. Daemon threads only to preserve what this pool already had: + // ForkJoinPool's workers are daemon, Executors.defaultThreadFactory()'s + // are not. + private static final ExecutorService POOL = Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "process-launcher-test-" + POOL_THREADS.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); private static final long FUTURE_TIMEOUT_SECONDS = 30; /** diff --git a/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java b/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java index 0e33ba0..a9dabf8 100644 --- a/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java +++ b/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java @@ -15,6 +15,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -28,19 +29,20 @@ * through plan()/apply() would download a terraform release - and that factory * is the single funnel every command path goes through. *

- * The multi-launcher test attaches no output listeners, and that is deliberate - * rather than lazy. launch() only submits the two stream reader tasks when a - * listener is set, and those readers block on the child's pipe for as long as it - * lives, so each live launcher costs three tasks on the client's work-stealing - * pool - whose parallelism is availableProcessors(). Three concurrent launchers - * is nine tasks: that saturates a 4-core runner and the last launcher's reader - * is never scheduled. It needs none of the child's output, so the readers are - * simply never created, and its readiness is signalled through the filesystem - * for the same reason. + * The multi-launcher test attaches no output listeners and signals child + * readiness through the filesystem. That was originally forced on it: the + * client's pool used to be sized to availableProcessors(), so three launchers - + * nine blocking tasks - saturated a 4-core runner and the third child's + * readiness line was never delivered. everyConcurrentCommandDeliversItsOutput + * below is the regression test for that, and the pool now grows to the blocking + * work, so the constraint is gone. The test keeps both choices anyway: it needs + * none of the child's output, and a filesystem handshake does not depend on + * reader scheduling at all, which is the right way round for a test whose + * subject is close() rather than output capture. *

- * The single-launcher tests do keep their listeners. Three tasks fit, and - * dropping them there would lose the only coverage of close() running against - * readers genuinely blocked in readLine() on a live pipe. + * The single-launcher tests do keep their listeners, deliberately: dropping them + * would lose the only coverage of close() running against readers genuinely + * blocked in readLine() on a live pipe. */ @DisabledOnOs(OS.WINDOWS) class TerraformClientTest { @@ -171,6 +173,72 @@ void closeTerminatesEveryLauncherInParallelEvenWhenInterrupted(@TempDir Path rea "launchers should cost one grace period between them, not one each, took " + elapsedMillis + "ms"); } + /** + * Regression test for the client's executor being sized for CPU work while + * every task it runs is blocking I/O. + *

+ * Each running command holds three tasks of that pool blocked for as long as + * the child lives: two stream readers and the waitFor() supplier. A + * ForkJoinPool only grows workers for a ManagedBlocker, and neither + * readLine() nor waitFor() is one, so a pool with parallelism + * availableProcessors() can never run more than that many of them. N + * concurrent commands need 3N threads, so starvation begins once + * 3N - 1 > P - measured on 4 cores, the second concurrent command already + * has a reader that is never scheduled. + *

+ * The consequence is not slow output, it is a stopped child: an unscheduled + * reader stops draining the pipe, the pipe buffer fills, and the child + * blocks in write(2) and makes no further progress. Measured against the + * unfixed pool, a child froze after 8 loop iterations and had not advanced + * 20 seconds later, with zero lines delivered. For a terraform apply that is + * a run wedged mid-flight because something else is running on the same + * client. + *

+ * The test sizes itself to the machine rather than pinning the core count + * with a JVM flag: it launches P/3 + 2 commands, which satisfies + * 3N - 1 > P for every P >= 1, so it starves on any host without the + * fix and passes on any host with it - while costing a third of what one + * command per core would on a large machine. Each child deliberately writes + * more than a pipe buffer to each stream - a quiet child goes dark without + * ever wedging - and then stays alive, so that the readers of the commands + * that did get scheduled stay blocked rather than releasing their workers. + */ + @Test + void everyConcurrentCommandDeliversItsOutput() throws Exception { + // seq 1 20000 is 108,894 bytes, 1.66x the 64 KiB pipe buffer. + final int lines = 20_000; + // Starvation needs 3N - 1 > P, and P/3 + 2 satisfies it at every P + // (P=1 -> N=2, P=4 -> 3, P=14 -> 6, P=64 -> 23) without putting one + // process per core on a large machine. + final int processors = Runtime.getRuntime().availableProcessors(); + final int commands = processors / 3 + 2; + TerraformClient client = TerraformClient.builder().build(); + List stdout = new ArrayList<>(); + List stderr = new ArrayList<>(); + try { + for (int i = 0; i < commands; i++) { + AtomicLong out = new AtomicLong(); + AtomicLong err = new AtomicLong(); + stdout.add(out); + stderr.add(err); + ProcessLauncher launcher = client.newLauncher(processData(0), "bash", "-c", + "seq 1 " + lines + "; seq 1 " + lines + " 1>&2; exec sleep 300"); + launcher.setOutputListener(line -> out.incrementAndGet()); + launcher.setErrorListener(line -> err.incrementAndGet()); + launcher.launch(); + } + + await(() -> stdout.stream().allMatch(c -> c.get() == lines) + && stderr.stream().allMatch(c -> c.get() == lines), + () -> "every concurrent command must have its output read, but with " + + commands + " commands on " + processors + " processors the delivered line counts were stdout=" + + stdout + " stderr=" + stderr + " of " + lines + + " each; a command showing 0 had no reader scheduled, and its child is blocked in write()"); + } finally { + client.close(); + } + } + /** * The timeout is configured per job on TerraformProcessData, so it has to * reach the launcher the client builds for that job. Without the wiring the @@ -194,11 +262,21 @@ void timeoutSecondsFromProcessDataIsAppliedToTheLauncher() throws Exception { } } - private static void await(java.util.function.BooleanSupplier condition, String message) throws InterruptedException { + private static void await(java.util.function.BooleanSupplier condition, + java.util.function.Supplier message) throws InterruptedException { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); while (!condition.getAsBoolean() && System.nanoTime() < deadline) { Thread.sleep(20); } assertTrue(condition.getAsBoolean(), message); } + + /** + * The message is a supplier because the counts it reports are mutating while + * the condition is being polled; building it eagerly would report the values + * from before the wait. + */ + private static void await(java.util.function.BooleanSupplier condition, String message) throws InterruptedException { + await(condition, () -> message); + } }