diff --git a/README.md b/README.md index 298466f..b58018a 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,17 @@ io.terrakube.terraform.flags.jsonOutput=true > This feature is only supported in terraform >= 0.15.3 +The downloads of the terraform/tofu releases list and binaries have timeouts, so a download that never completes fails instead of hanging. The defaults can be overridden: + +``` +# Max idle time between reads of a response +io.terrakube.terraform.flags.downloadResponseTimeout=60s +# Max time to download the releases list +io.terrakube.terraform.flags.releasesDownloadTimeout=2m +# Max time to download a binary archive +io.terrakube.terraform.flags.binaryDownloadTimeout=10m +``` + The final step is to let the Spring framework wire up everything in your spring boot application: Example 1: `Not thread safe` diff --git a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java index 7fa96e1..e56541b 100644 --- a/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java +++ b/terraform-client/src/main/java/io/terrakube/terraform/TerraformClient.java @@ -5,6 +5,7 @@ import org.apache.maven.artifact.versioning.ComparableVersion; import java.io.*; +import java.time.Duration; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.*; @@ -60,6 +61,10 @@ public class TerraformClient implements AutoCloseable { private String backendConfig; private String terraformReleasesUrl; private String tofuReleasesUrl; + // Download timeouts, null means the TerraformDownloader default. + private Duration downloadResponseTimeout; + private Duration releasesDownloadTimeout; + private Duration binaryDownloadTimeout; private String varFileName; @@ -554,7 +559,8 @@ public TerraformDownloader createTerraformDownloader() { String TOFU_RELEASES_URL = (this.tofuReleasesUrl != null && !tofuReleasesUrl.isEmpty()) ? this.tofuReleasesUrl : TerraformDownloader.TOFU_RELEASES_URL; log.info("Creating terraform downloader using terraform release URL: {} and tofu release URL: {}", TERRAFORM_RELEASES_URL, TOFU_RELEASES_URL); - return new TerraformDownloader(TERRAFORM_RELEASES_URL, TOFU_RELEASES_URL); + return new TerraformDownloader(TERRAFORM_RELEASES_URL, TOFU_RELEASES_URL, downloadResponseTimeout, + releasesDownloadTimeout, binaryDownloadTimeout); } } diff --git a/terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java b/terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java index d812f03..19f28f5 100644 --- a/terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java +++ b/terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java @@ -16,10 +16,12 @@ import org.springframework.core.io.buffer.DataBuffer; import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.http.MediaType; +import org.springframework.http.client.reactive.ClientHttpRequest; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.HttpClientRequest; import java.io.File; import java.io.FileInputStream; @@ -63,11 +65,33 @@ public class TerraformDownloader { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + // Without timeouts a download that never completes blocks the calling thread forever. For the releases + // list that also means holding RELEASES_LOCKS, so every other thread that needs the list hangs too. + // The response timeout is the max idle time between reads, the download timeouts cap the whole download. + // Connecting is already capped by Netty's default 30s connect timeout. + public static final Duration DEFAULT_RESPONSE_TIMEOUT = Duration.ofSeconds(60); + public static final Duration DEFAULT_RELEASES_DOWNLOAD_TIMEOUT = Duration.ofMinutes(2); + public static final Duration DEFAULT_BINARY_DOWNLOAD_TIMEOUT = Duration.ofMinutes(10); + + // One client per process. WebClient and the Reactor Netty HttpClient are thread safe, so there is + // no need to build new ones for every download. They live in a holder so they are only created on + // the first download, not when the class is loaded. + private static final class WebClients { + private static final HttpClient HTTP_CLIENT = createHttpClient(); + static final WebClient RELEASES = createWebClient(HTTP_CLIENT, "releases-downloader", + List.of(MediaType.APPLICATION_JSON)); + static final WebClient BINARY = createWebClient(HTTP_CLIENT, "terraform-downloader", + List.of(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)); + } + // Makes sure we only cache the binaries for exactly specified versions, not for ranges private static final Pattern EXACT_VERSION = Pattern.compile("^v?\\d+\\.\\d+\\.\\d+(?:[-+][0-9A-Za-z.+-]+)?$"); private final String terraformReleasesUrl; private final String tofuReleasesUrl; + private final Duration responseTimeout; + private final Duration releasesDownloadTimeout; + private final Duration binaryDownloadTimeout; private File terraformDownloadDirectory; private File tofuDownloadDirectory; @@ -92,6 +116,11 @@ boolean isFresh() { } } + @FunctionalInterface + private interface ReleasesFetcher { + void download(String releasesUrl, File releasesFile); + } + @FunctionalInterface private interface ReleasesParser { T parse(File releasesFile) throws IOException; @@ -102,12 +131,27 @@ public TerraformDownloader() { } public TerraformDownloader(String terraformReleasesUrl, String tofuReleasesUrl) { + this(terraformReleasesUrl, tofuReleasesUrl, null, null, null); + } + + /** + * @param responseTimeout max idle time between reads of a response, {@code null} for {@link #DEFAULT_RESPONSE_TIMEOUT} + * @param releasesDownloadTimeout max time to download a releases list, {@code null} for {@link #DEFAULT_RELEASES_DOWNLOAD_TIMEOUT} + * @param binaryDownloadTimeout max time to download a binary archive, {@code null} for {@link #DEFAULT_BINARY_DOWNLOAD_TIMEOUT} + */ + public TerraformDownloader(String terraformReleasesUrl, String tofuReleasesUrl, Duration responseTimeout, + Duration releasesDownloadTimeout, Duration binaryDownloadTimeout) { log.info("Initialize TerraformDownloader using custom URL"); log.debug("Terraform releases URL: {}", terraformReleasesUrl); log.debug("Tofu releases URL: {}", tofuReleasesUrl); this.terraformReleasesUrl = terraformReleasesUrl; this.tofuReleasesUrl = tofuReleasesUrl; + this.responseTimeout = responseTimeout != null ? responseTimeout : DEFAULT_RESPONSE_TIMEOUT; + this.releasesDownloadTimeout = releasesDownloadTimeout != null ? releasesDownloadTimeout : DEFAULT_RELEASES_DOWNLOAD_TIMEOUT; + this.binaryDownloadTimeout = binaryDownloadTimeout != null ? binaryDownloadTimeout : DEFAULT_BINARY_DOWNLOAD_TIMEOUT; + log.debug("Download timeouts: response {}, releases {}, binary {}", this.responseTimeout, + this.releasesDownloadTimeout, this.binaryDownloadTimeout); try { createDownloadTempDirectory(); @@ -160,7 +204,7 @@ private void createDownloadTofuTempDirectory() throws IOException { } private TerraformResponse getTerraformReleases() { - return getCachedReleases("terraform", terraformReleasesUrl, releasesFile -> { + return getCachedReleases("terraform", terraformReleasesUrl, this::downloadReleasesToFile, releasesFile -> { TerraformResponse response = OBJECT_MAPPER.readValue(releasesFile, TerraformResponse.class); if (response == null || response.getVersions() == null) { throw new IOException("Terraform releases response has no versions"); @@ -171,7 +215,7 @@ private TerraformResponse getTerraformReleases() { } private List getTofuReleases() { - return getCachedReleases("tofu", tofuReleasesUrl, releasesFile -> { + return getCachedReleases("tofu", tofuReleasesUrl, this::downloadReleasesToFile, releasesFile -> { List releases = OBJECT_MAPPER.readValue(releasesFile, OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, TofuRelease.class)); if (releases == null) { @@ -183,7 +227,8 @@ private List getTofuReleases() { } @SuppressWarnings("unchecked") - private static T getCachedReleases(String product, String releasesUrl, ReleasesParser parser) { + private static T getCachedReleases(String product, String releasesUrl, ReleasesFetcher fetcher, + ReleasesParser parser) { String key = product + "|" + releasesUrl; CachedReleases cached = (CachedReleases) RELEASES_CACHE.get(key); if (cached != null && cached.isFresh()) { @@ -199,10 +244,13 @@ private static T getCachedReleases(String product, String releasesUrl, Relea } try { - T releases = fetchReleases(product, releasesUrl, parser); + T releases = fetchReleases(product, releasesUrl, fetcher, parser); RELEASES_CACHE.put(key, new CachedReleases<>(releases, Instant.now().plus(RELEASES_CACHE_TTL))); return releases; - } catch (Exception e) { + } catch (Exception | OutOfMemoryError e) { + // OutOfMemoryError is caught on purpose: Netty throws io.netty.util.internal.OutOfDirectMemoryError + // when direct memory runs out. That says nothing about the heap, and failing here (or falling + // back to the previous list) is better than letting it escape and leaving the job in a broken state. if (cached != null) { log.warn("Error fetching {} releases from {}, using the previous list: {}", product, releasesUrl, e.getMessage()); RELEASES_CACHE.put(key, new CachedReleases<>(cached.releases(), Instant.now().plus(RELEASES_RETRY_AFTER_FAILURE))); @@ -213,13 +261,14 @@ private static T getCachedReleases(String product, String releasesUrl, Relea } } - private static T fetchReleases(String product, String releasesUrl, ReleasesParser parser) throws IOException { + private static T fetchReleases(String product, String releasesUrl, ReleasesFetcher fetcher, + ReleasesParser parser) throws IOException { log.info("Downloading {} releases list from {}", product, releasesUrl); Path tempDirectory = Files.createDirectories( Paths.get(FileUtils.getTempDirectory().getAbsolutePath(), UUID.randomUUID().toString())); try { File releasesFile = tempDirectory.resolve(product + "-releases.json").toFile(); - downloadReleasesToFile(releasesUrl, releasesFile); + fetcher.download(releasesUrl, releasesFile); return parser.parse(releasesFile); } finally { FileUtils.deleteQuietly(tempDirectory.toFile()); @@ -243,17 +292,26 @@ private static HttpClient createHttpClient() { .proxyWithSystemProperties(); } - private static void downloadReleasesToFile(String releasesUrl, File releasesFile) { - WebClient webClient = WebClient.builder() - .clientConnector(new ReactorClientHttpConnector(createHttpClient())) + private static WebClient createWebClient(HttpClient httpClient, String userAgent, List accept) { + return WebClient.builder() + .clientConnector(new ReactorClientHttpConnector(httpClient)) .defaultHeaders(h -> { - h.add("User-Agent", "releases-downloader"); - h.setAccept(List.of(MediaType.APPLICATION_JSON)); + h.add("User-Agent", userAgent); + h.setAccept(accept); }) .build(); + } + + // The response timeout is set per request, so the shared HttpClient can serve downloaders with different settings. + private void withResponseTimeout(ClientHttpRequest request) { + HttpClientRequest nativeRequest = request.getNativeRequest(); + nativeRequest.responseTimeout(responseTimeout); + } - webClient.get() + private void downloadReleasesToFile(String releasesUrl, File releasesFile) { + WebClients.RELEASES.get() .uri(releasesUrl) + .httpRequest(this::withResponseTimeout) .retrieve() .onStatus( status -> !status.is2xxSuccessful(), @@ -262,7 +320,7 @@ private static void downloadReleasesToFile(String releasesUrl, File releasesFile .bodyToFlux(DataBuffer.class) .as(dataBufferFlux -> DataBufferUtils.write(dataBufferFlux, releasesFile.toPath() )) .then() - .block(); + .block(releasesDownloadTimeout); } private String downloadFileOrReturnPathIfAlreadyExists(String fileName, String zipReleaseUrl, String version, @@ -289,17 +347,10 @@ private String downloadFileOrReturnPathIfAlreadyExists(String fileName, String z } log.info("Downloading {} from: {}", product, zipReleaseUrl); - WebClient webClient = WebClient.builder() - .clientConnector(new ReactorClientHttpConnector(createHttpClient())) - .defaultHeaders(h -> { - h.add("User-Agent", "terraform-downloader"); - h.setAccept(List.of(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)); - }) - .build(); - try { - webClient.get() + WebClients.BINARY.get() .uri(zipReleaseUrl) + .httpRequest(this::withResponseTimeout) .retrieve() .onStatus( status -> !status.is2xxSuccessful(), @@ -308,8 +359,8 @@ private String downloadFileOrReturnPathIfAlreadyExists(String fileName, String z .bodyToFlux(DataBuffer.class) .as(dataBufferFlux -> DataBufferUtils.write(dataBufferFlux, zipFile.toPath())) .then() - .block(); - } catch (RuntimeException exception) { + .block(binaryDownloadTimeout); + } catch (RuntimeException | OutOfMemoryError exception) { // A partial archive would otherwise be picked up as an existing archive next time. FileUtils.deleteQuietly(zipFile); throw new IOException("Unable to download ".concat(zipReleaseUrl), exception); diff --git a/terraform-client/src/test/java/io/terrakube/terraform/TerraformDownloaderTest.java b/terraform-client/src/test/java/io/terrakube/terraform/TerraformDownloaderTest.java index e9e4149..9c77f9e 100644 --- a/terraform-client/src/test/java/io/terrakube/terraform/TerraformDownloaderTest.java +++ b/terraform-client/src/test/java/io/terrakube/terraform/TerraformDownloaderTest.java @@ -17,12 +17,18 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -47,6 +53,9 @@ class TerraformDownloaderTest { private final AtomicInteger terraformRequests = new AtomicInteger(); private final AtomicInteger tofuRequests = new AtomicInteger(); private volatile int tofuStatus = 200; + private volatile boolean tofuHangs = false; + private final CountDownLatch releaseHangingRequests = new CountDownLatch(1); + private ExecutorService serverExecutor; @BeforeEach void setUp() throws IOException { @@ -61,14 +70,28 @@ void setUp() throws IOException { }); server.createContext("/tofu.json", exchange -> { tofuRequests.incrementAndGet(); + if (tofuHangs) { + // Never answer, like a fetch whose response never completes. + try { + releaseHangingRequests.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + exchange.close(); + return; + } respond(exchange, tofuStatus, tofuStatus == 200 ? TOFU_RELEASES : "{}"); }); + serverExecutor = Executors.newCachedThreadPool(); + server.setExecutor(serverExecutor); server.start(); } @AfterEach void tearDown() { + releaseHangingRequests.countDown(); server.stop(0); + serverExecutor.shutdownNow(); TerraformDownloader.clearReleasesCache(); System.setProperty("user.home", originalUserHome); } @@ -114,6 +137,43 @@ void failingFetchWithoutAPreviousListIsReported() { assertThrows(IllegalStateException.class, () -> downloader.resolveTofuVersion(">=1.0.0")); } + @Test + void hangingFetchWithoutAPreviousListFailsInsteadOfBlockingForever() { + tofuHangs = true; + TerraformDownloader downloader = downloaderWithShortReleasesTimeout(); + + assertTimeoutPreemptively(Duration.ofSeconds(10), () -> + assertThrows(IllegalStateException.class, () -> downloader.resolveTofuVersion(">=1.0.0"))); + } + + @Test + void previousListIsUsedWhenARefreshHangs() { + TerraformDownloader downloader = downloaderWithShortReleasesTimeout(); + downloader.resolveTofuVersion(">=1.0.0"); + + TerraformDownloader.expireReleasesCache(); + tofuHangs = true; + + assertTimeoutPreemptively(Duration.ofSeconds(10), () -> + assertEquals("1.12.4", downloader.resolveTofuVersion(">=1.0.0"))); + assertEquals(2, tofuRequests.get()); + } + + @Test + void hangingResponseIsCutByTheResponseTimeout() { + tofuHangs = true; + // Only the response timeout is short, so this proves it is applied to the request. + TerraformDownloader downloader = new TerraformDownloader(terraformUrl(), tofuUrl(), + Duration.ofMillis(500), Duration.ofMinutes(5), null); + + assertTimeoutPreemptively(Duration.ofSeconds(10), () -> + assertThrows(IllegalStateException.class, () -> downloader.resolveTofuVersion(">=1.0.0"))); + } + + private TerraformDownloader downloaderWithShortReleasesTimeout() { + return new TerraformDownloader(terraformUrl(), tofuUrl(), null, Duration.ofMillis(500), null); + } + @Test @DisabledOnOs(OS.WINDOWS) void installedBinaryIsUsedWithoutFetchingReleasesOrDownloading() throws IOException { diff --git a/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformAutoConfiguration.java b/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformAutoConfiguration.java index 1287579..7a49c5d 100644 --- a/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformAutoConfiguration.java +++ b/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformAutoConfiguration.java @@ -20,6 +20,9 @@ public TerraformClient terraformClient(@NonNull TerraformProperties tfProperties .jsonOutput(tfProperties.isJsonOutput()) .terraformReleasesUrl(tfProperties.getTerraformReleasesUrl()) .tofuReleasesUrl(tfProperties.getTofuReleasesUrl()) + .downloadResponseTimeout(tfProperties.getDownloadResponseTimeout()) + .releasesDownloadTimeout(tfProperties.getReleasesDownloadTimeout()) + .binaryDownloadTimeout(tfProperties.getBinaryDownloadTimeout()) .build(); } } diff --git a/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformProperties.java b/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformProperties.java index ad70698..e76efca 100644 --- a/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformProperties.java +++ b/terraform-spring-boot-autoconfigure/src/main/java/io/terrakube/terraform/spring/autoconfigure/TerraformProperties.java @@ -6,6 +6,8 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; +import java.time.Duration; + @Component @Getter @Setter @@ -18,4 +20,12 @@ public class TerraformProperties { private boolean jsonOutput; private String terraformReleasesUrl; private String tofuReleasesUrl; + // Same defaults as TerraformDownloader.DEFAULT_*. Not referenced directly so that loading these + // properties does not load TerraformDownloader, which needs webflux on the classpath. + // Max idle time between reads while downloading the releases list or a binary, e.g. 60s. + private Duration downloadResponseTimeout = Duration.ofSeconds(60); + // Max time to download the terraform/tofu releases list, e.g. 2m. + private Duration releasesDownloadTimeout = Duration.ofMinutes(2); + // Max time to download a terraform/tofu binary archive, e.g. 10m. + private Duration binaryDownloadTimeout = Duration.ofMinutes(10); }