diff --git a/mooncake-store/src/utils/s3_helper.cpp b/mooncake-store/src/utils/s3_helper.cpp index f951d4bbee..d26f37b594 100644 --- a/mooncake-store/src/utils/s3_helper.cpp +++ b/mooncake-store/src/utils/s3_helper.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include "utils/type_util.h" #include "fmt/format.h" @@ -50,6 +52,14 @@ struct S3Env { bool use_virtual_addressing = true; + bool use_https = true; + + Aws::Client::RequestChecksumCalculation request_checksum = + Aws::Client::RequestChecksumCalculation::WHEN_SUPPORTED; + + Aws::Client::ResponseChecksumValidation response_checksum = + Aws::Client::ResponseChecksumValidation::WHEN_SUPPORTED; + int64_t connect_timeout_ms = kDefaultS3ConnectTimeoutMs; int64_t request_timeout_ms = kDefaultS3RequestTimeoutMs; @@ -92,6 +102,20 @@ void AssignTimeoutFromEnv(const char *env_name, int64_t default_value, target = default_value; } +template +T ParseChecksumMode(const std::string &value) { + std::string lower = value; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + if (lower == "when_required") return T::WHEN_REQUIRED; + if (lower == "when_supported") return T::WHEN_SUPPORTED; + if (!lower.empty()) { + LOG(WARNING) << "Invalid value: " << value + << ", falling back to when_supported"; + } + return T::WHEN_SUPPORTED; +} + } // namespace bool S3Helper::aws_initialized = false; @@ -113,6 +137,21 @@ void S3Helper::InitAPI() { AssignBoolFromEnv("MOONCAKE_AWS_USE_VIRTUAL_ADDRESSING", s3_env.use_virtual_addressing); + AssignBoolFromEnv("MOONCAKE_AWS_S3_USE_HTTPS", s3_env.use_https); + + { + std::string tmp; + AssignStringFromEnv("MOONCAKE_AWS_S3_REQUEST_CHECKSUM", tmp); + s3_env.request_checksum = + ParseChecksumMode(tmp); + } + { + std::string tmp; + AssignStringFromEnv("MOONCAKE_AWS_S3_RESPONSE_CHECKSUM", tmp); + s3_env.response_checksum = + ParseChecksumMode(tmp); + } + AssignTimeoutFromEnv("MOONCAKE_AWS_CONNECT_TIMEOUT_MS", kDefaultS3ConnectTimeoutMs, s3_env.connect_timeout_ms); AssignTimeoutFromEnv("MOONCAKE_AWS_REQUEST_TIMEOUT_MS", @@ -132,7 +171,10 @@ S3Helper::S3Helper(const std::string &endpoint, const std::string &bucket, config.connectTimeoutMs = s3_env.connect_timeout_ms; config.requestTimeoutMs = s3_env.request_timeout_ms; - config.scheme = Aws::Http::Scheme::HTTPS; + config.scheme = + s3_env.use_https ? Aws::Http::Scheme::HTTPS : Aws::Http::Scheme::HTTP; + config.checksumConfig.requestChecksumCalculation = s3_env.request_checksum; + config.checksumConfig.responseChecksumValidation = s3_env.response_checksum; if (!region.empty()) { config.region = region; @@ -872,4 +914,4 @@ tl::expected S3Helper::DeleteObjectsWithPrefix( return DeleteObjects(object_keys); } -} // namespace mooncake \ No newline at end of file +} // namespace mooncake