diff --git a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java index 10beee6..1ad5f22 100644 --- a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java +++ b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java @@ -41,7 +41,25 @@ public class TerraformClient implements AutoCloseable { // spend". They happen to agree today. private static final long CLOSE_GRACE_PERIOD_SECONDS = 10; - private final ExecutorService executor = Executors.newWorkStealingPool(); + private static final AtomicInteger CLIENT_THREAD_COUNT = new AtomicInteger(); + + // A cached pool, not newWorkStealingPool(). Every task submitted here is + // blocking I/O - two pipe readers and a waitFor() per command - and a + // ForkJoinPool sized to availableProcessors() only grows workers for a + // ManagedBlocker, which neither readLine() nor waitFor() is. So N concurrent + // commands need 3N blocked threads and can never have more than + // availableProcessors(): measured on 4 cores, the second concurrent command + // already has a starved reader, and once that reader's pipe buffer fills the + // child itself blocks on write and stops making progress. A cached pool + // grows to the blocking work and reaps idle threads after 60s. + private final ExecutorService executor = Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "terraform-client-" + CLIENT_THREAD_COUNT.incrementAndGet()); + // Daemon, as ForkJoinPool's workers were, so the library never holds JVM + // exit open. Pinned classloader for the same reason as TERMINATOR's. + thread.setDaemon(true); + thread.setContextClassLoader(TerraformClient.class.getClassLoader()); + return thread; + }); // Final with an initializer and no accessors on purpose: that is what keeps // these two off the generated builder, off the all-args constructor and out // of the published API. A plain volatile boolean for the flag gave Lombok a diff --git a/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java b/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java index d9fd334..00255d1 100644 --- a/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java +++ b/terraform-client/src/test/java/io/terrakube/terraform/ProcessLauncherTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; @@ -30,7 +31,20 @@ @DisabledOnOs(OS.WINDOWS) class ProcessLauncherTest { - private static final ExecutorService POOL = Executors.newWorkStealingPool(); + private static final AtomicInteger POOL_THREADS = new AtomicInteger(); + + // Cached, not work-stealing, for the same reason the client's own pool is: + // every task here is blocking I/O, and a ForkJoinPool sized to + // availableProcessors() cannot run more than that many blocked tasks, so + // errorStreamIsFullyCapturedWhenExitCodeFutureCompletes below deadlocks at + // one core. Daemon threads only to preserve what this pool already had: + // ForkJoinPool's workers are daemon, Executors.defaultThreadFactory()'s + // are not. + private static final ExecutorService POOL = Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "process-launcher-test-" + POOL_THREADS.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); private static final long FUTURE_TIMEOUT_SECONDS = 30; /** diff --git a/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java b/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java index 0e33ba0..a9dabf8 100644 --- a/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java +++ b/terraform-client/src/test/java/io/terrakube/terraform/TerraformClientTest.java @@ -15,6 +15,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -28,19 +29,20 @@ * through plan()/apply() would download a terraform release - and that factory * is the single funnel every command path goes through. *
- * The multi-launcher test attaches no output listeners, and that is deliberate - * rather than lazy. launch() only submits the two stream reader tasks when a - * listener is set, and those readers block on the child's pipe for as long as it - * lives, so each live launcher costs three tasks on the client's work-stealing - * pool - whose parallelism is availableProcessors(). Three concurrent launchers - * is nine tasks: that saturates a 4-core runner and the last launcher's reader - * is never scheduled. It needs none of the child's output, so the readers are - * simply never created, and its readiness is signalled through the filesystem - * for the same reason. + * The multi-launcher test attaches no output listeners and signals child + * readiness through the filesystem. That was originally forced on it: the + * client's pool used to be sized to availableProcessors(), so three launchers - + * nine blocking tasks - saturated a 4-core runner and the third child's + * readiness line was never delivered. everyConcurrentCommandDeliversItsOutput + * below is the regression test for that, and the pool now grows to the blocking + * work, so the constraint is gone. The test keeps both choices anyway: it needs + * none of the child's output, and a filesystem handshake does not depend on + * reader scheduling at all, which is the right way round for a test whose + * subject is close() rather than output capture. *
- * The single-launcher tests do keep their listeners. Three tasks fit, and - * dropping them there would lose the only coverage of close() running against - * readers genuinely blocked in readLine() on a live pipe. + * The single-launcher tests do keep their listeners, deliberately: dropping them + * would lose the only coverage of close() running against readers genuinely + * blocked in readLine() on a live pipe. */ @DisabledOnOs(OS.WINDOWS) class TerraformClientTest { @@ -171,6 +173,72 @@ void closeTerminatesEveryLauncherInParallelEvenWhenInterrupted(@TempDir Path rea "launchers should cost one grace period between them, not one each, took " + elapsedMillis + "ms"); } + /** + * Regression test for the client's executor being sized for CPU work while + * every task it runs is blocking I/O. + *
+ * Each running command holds three tasks of that pool blocked for as long as + * the child lives: two stream readers and the waitFor() supplier. A + * ForkJoinPool only grows workers for a ManagedBlocker, and neither + * readLine() nor waitFor() is one, so a pool with parallelism + * availableProcessors() can never run more than that many of them. N + * concurrent commands need 3N threads, so starvation begins once + * 3N - 1 > P - measured on 4 cores, the second concurrent command already + * has a reader that is never scheduled. + *
+ * The consequence is not slow output, it is a stopped child: an unscheduled + * reader stops draining the pipe, the pipe buffer fills, and the child + * blocks in write(2) and makes no further progress. Measured against the + * unfixed pool, a child froze after 8 loop iterations and had not advanced + * 20 seconds later, with zero lines delivered. For a terraform apply that is + * a run wedged mid-flight because something else is running on the same + * client. + *
+ * The test sizes itself to the machine rather than pinning the core count
+ * with a JVM flag: it launches P/3 + 2 commands, which satisfies
+ * 3N - 1 > P for every P >= 1, so it starves on any host without the
+ * fix and passes on any host with it - while costing a third of what one
+ * command per core would on a large machine. Each child deliberately writes
+ * more than a pipe buffer to each stream - a quiet child goes dark without
+ * ever wedging - and then stays alive, so that the readers of the commands
+ * that did get scheduled stay blocked rather than releasing their workers.
+ */
+ @Test
+ void everyConcurrentCommandDeliversItsOutput() throws Exception {
+ // seq 1 20000 is 108,894 bytes, 1.66x the 64 KiB pipe buffer.
+ final int lines = 20_000;
+ // Starvation needs 3N - 1 > P, and P/3 + 2 satisfies it at every P
+ // (P=1 -> N=2, P=4 -> 3, P=14 -> 6, P=64 -> 23) without putting one
+ // process per core on a large machine.
+ final int processors = Runtime.getRuntime().availableProcessors();
+ final int commands = processors / 3 + 2;
+ TerraformClient client = TerraformClient.builder().build();
+ List