diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index 758509f5..4bf1f9df 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -97,6 +97,8 @@ locals { max_instances = var.datacommons_services_max_instances cpu = var.datacommons_services_cpu memory = var.datacommons_services_memory + cpu_idle = var.datacommons_services_cpu_idle + startup_cpu_boost = var.datacommons_services_startup_cpu_boost google_analytics_tag = var.datacommons_services_google_analytics_tag_id enable_mcp = var.datacommons_services_enable_mcp search_scope = var.datacommons_services_mcp_search_scope @@ -121,8 +123,10 @@ locals { location_id = var.redis_location_id alternative_location_id = var.redis_alternative_location_id replica_count = var.redis_replica_count - vpc_network_name = var.redis_vpc_network_name - vpc_connector_cidr = var.redis_vpc_connector_cidr + vpc_network_name = var.redis_vpc_network_name + vpc_connector_cidr = var.redis_vpc_connector_cidr + vpc_connector_min_instances = var.redis_vpc_connector_min_instances + vpc_connector_max_instances = var.redis_vpc_connector_max_instances } ingestion_config = { @@ -149,6 +153,8 @@ locals { # Workflow & Helper Service workflow_lock_acquisition_timeout = var.ingestion_workflow_lock_acquisition_timeout helper_service_image = coalesce(var.ingestion_helper_service_image, "gcr.io/datcom-ci/datacommons-ingestion-helper:${var.dcp_version}") + helper_service_cpu_idle = var.ingestion_helper_service_cpu_idle + helper_service_startup_cpu_boost = var.ingestion_helper_service_startup_cpu_boost # Dataflow Network & Scaling Configuration dataflow_ip_configuration = var.ingestion_dataflow_ip_configuration diff --git a/infra/dcp/modules/datacommons_services/main.tf b/infra/dcp/modules/datacommons_services/main.tf index 35cf9289..3a9978b1 100644 --- a/infra/dcp/modules/datacommons_services/main.tf +++ b/infra/dcp/modules/datacommons_services/main.tf @@ -40,6 +40,8 @@ resource "google_cloud_run_v2_service" "dc_web_service" { containers { image = var.image resources { + cpu_idle = var.cpu_idle + startup_cpu_boost = var.startup_cpu_boost limits = { cpu = var.cpu memory = var.memory diff --git a/infra/dcp/modules/datacommons_services/variables.tf b/infra/dcp/modules/datacommons_services/variables.tf index 60773c9e..136c138f 100644 --- a/infra/dcp/modules/datacommons_services/variables.tf +++ b/infra/dcp/modules/datacommons_services/variables.tf @@ -35,6 +35,18 @@ variable "memory" { type = string } +variable "cpu_idle" { + type = bool + description = "When true, CPU is only allocated during request processing (cheaper for low-traffic services). When false, CPU is always allocated (better performance, avoids cold starts)." + default = false +} + +variable "startup_cpu_boost" { + type = bool + description = "Temporarily boost CPU allocation during container startup to reduce cold start latency." + default = true +} + variable "min_instances" { type = number } diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf index af90abc3..a09d024b 100644 --- a/infra/dcp/modules/ingestion/helper_service/main.tf +++ b/infra/dcp/modules/ingestion/helper_service/main.tf @@ -18,6 +18,10 @@ resource "google_cloud_run_v2_service" "ingestion_helper" { timeout = "1800s" containers { image = var.image + resources { + cpu_idle = var.cpu_idle + startup_cpu_boost = var.startup_cpu_boost + } env { name = "PROJECT_ID" diff --git a/infra/dcp/modules/ingestion/helper_service/variables.tf b/infra/dcp/modules/ingestion/helper_service/variables.tf index fdea683f..bcde0533 100644 --- a/infra/dcp/modules/ingestion/helper_service/variables.tf +++ b/infra/dcp/modules/ingestion/helper_service/variables.tf @@ -71,6 +71,20 @@ variable "ingestion_artifacts_path" { description = "Path where pre-processed files are placed for the next stage" } +variable "cpu_idle" { + type = bool + description = "When true, CPU is only allocated during request processing (cheaper for low-traffic services). When false, CPU is always allocated." + default = false + nullable = false +} + +variable "startup_cpu_boost" { + type = bool + description = "Temporarily boost CPU allocation during container startup to reduce cold start latency." + default = true + nullable = false +} + variable "skip_container_restarts" { type = bool description = "Set to true to skip updating container restart timestamps, speeding up terraform apply when container images have not changed." diff --git a/infra/dcp/modules/redis/main.tf b/infra/dcp/modules/redis/main.tf index 637b4ca6..28e0c65c 100644 --- a/infra/dcp/modules/redis/main.tf +++ b/infra/dcp/modules/redis/main.tf @@ -27,6 +27,6 @@ resource "google_vpc_access_connector" "connector" { region = var.region network = var.vpc_network_id ip_cidr_range = var.vpc_connector_cidr - min_instances = 2 - max_instances = 10 + min_instances = var.vpc_connector_min_instances + max_instances = var.vpc_connector_max_instances } diff --git a/infra/dcp/modules/redis/variables.tf b/infra/dcp/modules/redis/variables.tf index a74e1ea1..22b81048 100644 --- a/infra/dcp/modules/redis/variables.tf +++ b/infra/dcp/modules/redis/variables.tf @@ -44,3 +44,15 @@ variable "enable_connector" { type = bool default = true } + +variable "vpc_connector_min_instances" { + type = number + description = "Minimum number of VPC Access Connector instances." + default = 2 +} + +variable "vpc_connector_max_instances" { + type = number + description = "Maximum number of VPC Access Connector instances." + default = 10 +} diff --git a/infra/dcp/modules/stack/main.tf b/infra/dcp/modules/stack/main.tf index af485f50..1a84b298 100644 --- a/infra/dcp/modules/stack/main.tf +++ b/infra/dcp/modules/stack/main.tf @@ -199,6 +199,8 @@ module "ingestion_helper_service" { redis_port = var.redis_config.enable && length(module.redis) > 0 ? tostring(module.redis[0].redis_port) : "" ingestion_artifacts_path = "${var.ingestion_config.ingestion_artifacts_path}/metadata" skip_container_restarts = var.global.skip_container_restarts + cpu_idle = var.ingestion_config.helper_service_cpu_idle + startup_cpu_boost = var.ingestion_config.helper_service_startup_cpu_boost } @@ -245,8 +247,10 @@ module "redis" { alternative_location_id = var.redis_config.alternative_location_id replica_count = var.redis_config.replica_count vpc_network_id = data.google_compute_network.default.id - vpc_connector_cidr = var.redis_config.vpc_connector_cidr - enable_connector = true + vpc_connector_cidr = var.redis_config.vpc_connector_cidr + vpc_connector_min_instances = var.redis_config.vpc_connector_min_instances + vpc_connector_max_instances = var.redis_config.vpc_connector_max_instances + enable_connector = true } module "auth" { @@ -270,6 +274,8 @@ module "datacommons_services" { image = var.datacommons_services_config.image cpu = var.datacommons_services_config.cpu memory = var.datacommons_services_config.memory + cpu_idle = var.datacommons_services_config.cpu_idle + startup_cpu_boost = var.datacommons_services_config.startup_cpu_boost min_instances = var.datacommons_services_config.min_instances max_instances = var.datacommons_services_config.max_instances make_public = var.datacommons_services_config.allow_unauthenticated_access diff --git a/infra/dcp/modules/stack/variables.tf b/infra/dcp/modules/stack/variables.tf index d155e917..3dc2467f 100644 --- a/infra/dcp/modules/stack/variables.tf +++ b/infra/dcp/modules/stack/variables.tf @@ -45,6 +45,8 @@ variable "datacommons_services_config" { max_instances = number cpu = string memory = string + cpu_idle = optional(bool, false) + startup_cpu_boost = optional(bool, true) google_analytics_tag = string enable_mcp = bool search_scope = string @@ -69,15 +71,17 @@ variable "auth_config" { variable "redis_config" { type = object({ - enable = bool - instance_name = string - memory_size_gb = number - tier = string - location_id = string - alternative_location_id = string - replica_count = number - vpc_network_name = string - vpc_connector_cidr = string + enable = bool + instance_name = string + memory_size_gb = number + tier = string + location_id = string + alternative_location_id = string + replica_count = number + vpc_network_name = string + vpc_connector_cidr = string + vpc_connector_min_instances = number + vpc_connector_max_instances = number }) } @@ -106,6 +110,8 @@ variable "ingestion_config" { # Workflow & Helper Service workflow_lock_acquisition_timeout = number helper_service_image = optional(string) + helper_service_cpu_idle = optional(bool) + helper_service_startup_cpu_boost = optional(bool) # Dataflow network configuration # Use WORKER_IP_PRIVATE when a compute.vmExternalIpAccess org policy diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf index 3b0c42a3..7cd07302 100644 --- a/infra/dcp/variables.tf +++ b/infra/dcp/variables.tf @@ -151,6 +151,18 @@ variable "redis_vpc_connector_cidr" { default = "10.13.0.0/28" } +variable "redis_vpc_connector_min_instances" { + description = "Minimum number of VPC Access Connector instances. Lower values reduce baseline cost." + type = number + default = 2 +} + +variable "redis_vpc_connector_max_instances" { + description = "Maximum number of VPC Access Connector instances." + type = number + default = 10 +} + # ============================================================================= # Spanner Module # ============================================================================= @@ -279,6 +291,18 @@ variable "datacommons_services_memory" { default = "16G" } +variable "datacommons_services_cpu_idle" { + description = "When true, CPU is only allocated during request processing (cheaper for low-traffic services). When false, CPU is always allocated (better performance, avoids cold starts within running instances)." + type = bool + default = false +} + +variable "datacommons_services_startup_cpu_boost" { + description = "Temporarily boost CPU allocation during container startup to reduce cold start latency." + type = bool + default = true +} + variable "datacommons_services_allow_unauthenticated_access" { description = "Allow unauthenticated access to the public-facing services of the Data Commons Platform" type = bool @@ -428,6 +452,18 @@ variable "ingestion_helper_service_image" { default = null } +variable "ingestion_helper_service_cpu_idle" { + description = "When true, CPU is only allocated during request processing (cheaper for low-traffic services). When false, CPU is always allocated." + type = bool + default = null +} + +variable "ingestion_helper_service_startup_cpu_boost" { + description = "Temporarily boost CPU allocation during container startup to reduce cold start latency." + type = bool + default = null +} + # ============================================================================= # Ingestion - Dataflow Network Configuration # ============================================================================= @@ -500,3 +536,10 @@ check "ingestion_dataflow_workers_limits" { } } +check "redis_vpc_connector_instances_limits" { + assert { + condition = var.redis_vpc_connector_max_instances >= var.redis_vpc_connector_min_instances + error_message = "The redis_vpc_connector_max_instances must be greater than or equal to redis_vpc_connector_min_instances." + } +} +