Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions terraform-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,17 @@
</dependency>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
<scope>provided</scope>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-http</artifactId>
<exclusions>
<!-- ~12 MB of QUIC natives for HTTP/3, which the downloader never uses. netty-codec-http3 and
netty-codec-classes-quic must stay: HttpClientSecure checks Quic.isAvailable() when it loads,
which then just returns false. -->
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec-native-quic</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,22 @@
import org.semver4j.Semver;
import org.semver4j.range.RangeList;
import org.semver4j.range.RangeListFactory;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.MediaType;
import org.springframework.http.client.reactive.ClientHttpRequest;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.timeout.ReadTimeoutException;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import reactor.netty.http.client.HttpClient;
import reactor.netty.http.client.HttpClientRequest;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
Expand Down Expand Up @@ -73,15 +72,11 @@
public static final Duration DEFAULT_RELEASES_DOWNLOAD_TIMEOUT = Duration.ofMinutes(2);
public static final Duration DEFAULT_BINARY_DOWNLOAD_TIMEOUT = Duration.ofMinutes(10);

// One client per process. WebClient and the Reactor Netty HttpClient are thread safe, so there is
// no need to build new ones for every download. They live in a holder so they are only created on
// the first download, not when the class is loaded.
private static final class WebClients {
private static final HttpClient HTTP_CLIENT = createHttpClient();
static final WebClient RELEASES = createWebClient(HTTP_CLIENT, "releases-downloader",
List.of(MediaType.APPLICATION_JSON));
static final WebClient BINARY = createWebClient(HTTP_CLIENT, "terraform-downloader",
List.of(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL));
// One client per process. The Reactor Netty HttpClient is thread safe, so there is no need to build a new
// one for every download. It lives in a holder so it is only created on the first download, not when the
// class is loaded. Reactor Netty's default event loops use daemon threads, so it does not keep the JVM alive.
private static final class HttpClients {
static final HttpClient CLIENT = createHttpClient();
}

// Makes sure we only cache the binaries for exactly specified versions, not for ranges
Expand All @@ -92,6 +87,8 @@
private final Duration responseTimeout;
private final Duration releasesDownloadTimeout;
private final Duration binaryDownloadTimeout;
// null means the shared client, only the tests pass their own.
private final HttpClient httpClient;
private File terraformDownloadDirectory;

private File tofuDownloadDirectory;
Expand All @@ -118,7 +115,7 @@

@FunctionalInterface
private interface ReleasesFetcher {
void download(String releasesUrl, File releasesFile);
void download(String releasesUrl, File releasesFile) throws IOException;
}

@FunctionalInterface
Expand All @@ -141,6 +138,12 @@
*/
public TerraformDownloader(String terraformReleasesUrl, String tofuReleasesUrl, Duration responseTimeout,
Duration releasesDownloadTimeout, Duration binaryDownloadTimeout) {
this(terraformReleasesUrl, tofuReleasesUrl, responseTimeout, releasesDownloadTimeout, binaryDownloadTimeout, null);
}

// For the tests: the shared client reads the proxy system properties once, when it is built.
TerraformDownloader(String terraformReleasesUrl, String tofuReleasesUrl, Duration responseTimeout,
Duration releasesDownloadTimeout, Duration binaryDownloadTimeout, HttpClient httpClient) {
log.info("Initialize TerraformDownloader using custom URL");
log.debug("Terraform releases URL: {}", terraformReleasesUrl);
log.debug("Tofu releases URL: {}", tofuReleasesUrl);
Expand All @@ -150,6 +153,7 @@
this.responseTimeout = responseTimeout != null ? responseTimeout : DEFAULT_RESPONSE_TIMEOUT;
this.releasesDownloadTimeout = releasesDownloadTimeout != null ? releasesDownloadTimeout : DEFAULT_RELEASES_DOWNLOAD_TIMEOUT;
this.binaryDownloadTimeout = binaryDownloadTimeout != null ? binaryDownloadTimeout : DEFAULT_BINARY_DOWNLOAD_TIMEOUT;
this.httpClient = httpClient;
log.debug("Download timeouts: response {}, releases {}, binary {}", this.responseTimeout,
this.releasesDownloadTimeout, this.binaryDownloadTimeout);

Expand Down Expand Up @@ -285,42 +289,84 @@
}
// End tests

private static HttpClient createHttpClient() {
// proxyWithSystemProperties() reads http(s).proxyHost/proxyPort/proxyUser/proxyPassword, http.nonProxyHosts
// and socksProxyHost once, when the client is built. For the shared client that is the first download.
static HttpClient createHttpClient() {
return HttpClient.create()
.httpResponseDecoder(spec -> spec.maxHeaderSize(MAX_RESPONSE_HEADER_SIZE))
.followRedirect(true)
.proxyWithSystemProperties();
}

private static WebClient createWebClient(HttpClient httpClient, String userAgent, List<MediaType> accept) {
return WebClient.builder()
.clientConnector(new ReactorClientHttpConnector(httpClient))
.defaultHeaders(h -> {
h.add("User-Agent", userAgent);
h.setAccept(accept);
})
.build();
private void downloadReleasesToFile(String releasesUrl, File releasesFile) throws IOException {
downloadToFile(releasesUrl, releasesFile, "releases-downloader", "application/json", releasesDownloadTimeout);
}

// The response timeout is set per request, so the shared HttpClient can serve downloaders with different settings.
private void withResponseTimeout(ClientHttpRequest request) {
HttpClientRequest nativeRequest = request.getNativeRequest();
nativeRequest.responseTimeout(responseTimeout);
/**
* Streams the response body to the file. Netty's response timeout cuts a response that sends no data for
* {@code responseTimeout}, and the whole download is cancelled after {@code downloadTimeout}, so a stuck
* server can never block the caller (or the RELEASES_LOCKS it holds). Cancelling closes the connection.
*/
private void downloadToFile(String url, File file, String userAgent, String accept, Duration downloadTimeout)

Check failure on line 310 in terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 19 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=AzBuilder_terraform-spring-boot&issues=AaHsvmDJSLClxAc1NMwT&open=AaHsvmDJSLClxAc1NMwT&pullRequest=117
throws IOException {
// Derived per call so the shared client can serve downloaders with different settings.
HttpClient client = (httpClient != null ? httpClient : HttpClients.CLIENT)
.responseTimeout(responseTimeout)
.headers(h -> h.set(HttpHeaderNames.USER_AGENT, userAgent).set(HttpHeaderNames.ACCEPT, accept));

Mono<Void> download = client.get()
.uri(url)
.response((response, body) -> {
int status = response.status().code();
// An error page is not a releases list or an archive, so it is never written to the file.
if (status < 200 || status >= 300) {
return Mono.<Void>error(new IOException(String.format("GET %s failed with HTTP status %d", url, status)));
}
return Mono.using(
() -> FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING),
// Each ByteBuf is written within onNext; Reactor Netty releases it once onNext returns.
channel -> body.handle((buffer, sink) -> {
try {
while (buffer.isReadable()) {
buffer.readBytes(channel, buffer.readableBytes());
}
} catch (IOException e) {
sink.error(e);
}
}).then(),
TerraformDownloader::closeQuietly);
})
.then()
// Cancelling on timeout disposes the request, which closes the connection.
.timeout(downloadTimeout, Mono.error(() -> new IOException(
String.format("Download of %s did not complete within %s", url, downloadTimeout))));

try {
download.block();
} catch (RuntimeException e) {
Throwable cause = Exceptions.unwrap(e);
if (cause instanceof InterruptedException) {
// block() cancels the download when interrupted.
Thread.currentThread().interrupt();
throw new InterruptedIOException("Interrupted while downloading " + url);
}
if (cause instanceof IOException) {

Check warning on line 354 in terraform-client/src/main/java/io/terrakube/terraform/TerraformDownloader.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace this instanceof check and cast with 'instanceof IOException ioexception'

See more on https://sonarcloud.io/project/issues?id=AzBuilder_terraform-spring-boot&issues=AaHsvmDJSLClxAc1NMwU&open=AaHsvmDJSLClxAc1NMwU&pullRequest=117
throw (IOException) cause;
}
if (cause instanceof ReadTimeoutException) {
throw new IOException(String.format("No data received from %s for %s", url, responseTimeout), cause);
}
throw new IOException("Unable to download " + url, cause);
}
}

private void downloadReleasesToFile(String releasesUrl, File releasesFile) {
WebClients.RELEASES.get()
.uri(releasesUrl)
.httpRequest(this::withResponseTimeout)
.retrieve()
.onStatus(
status -> !status.is2xxSuccessful(),
clientResponse -> clientResponse.createException().flatMap(Mono::error)
)
.bodyToFlux(DataBuffer.class)
.as(dataBufferFlux -> DataBufferUtils.write(dataBufferFlux, releasesFile.toPath() ))
.then()
.block(releasesDownloadTimeout);
private static void closeQuietly(FileChannel channel) {
try {
channel.close();
} catch (IOException e) {
log.warn("Unable to close {}: {}", channel, e.getMessage());
}
}

private String downloadFileOrReturnPathIfAlreadyExists(String fileName, String zipReleaseUrl, String version,
Expand Down Expand Up @@ -348,19 +394,9 @@

log.info("Downloading {} from: {}", product, zipReleaseUrl);
try {
WebClients.BINARY.get()
.uri(zipReleaseUrl)
.httpRequest(this::withResponseTimeout)
.retrieve()
.onStatus(
status -> !status.is2xxSuccessful(),
clientResponse -> clientResponse.createException().flatMap(Mono::error)
)
.bodyToFlux(DataBuffer.class)
.as(dataBufferFlux -> DataBufferUtils.write(dataBufferFlux, zipFile.toPath()))
.then()
.block(binaryDownloadTimeout);
} catch (RuntimeException | OutOfMemoryError exception) {
downloadToFile(zipReleaseUrl, zipFile, "terraform-downloader", "application/octet-stream, */*",
binaryDownloadTimeout);
} catch (IOException | RuntimeException | OutOfMemoryError exception) {
// A partial archive would otherwise be picked up as an existing archive next time.
FileUtils.deleteQuietly(zipFile);
throw new IOException("Unable to download ".concat(zipReleaseUrl), exception);
Expand Down
Loading
Loading