diff --git a/docker/.env.example b/docker/.env.example index c708a40c15960f..54b79a32c0700f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,6 +1,7 @@ # ------------------------------------------------------------------ # Essential defaults for Docker Compose deployments. # Only include variables required for services to start. +# Do not add optional variables to this file. # # For a default deployment, copy this file to .env and run: # docker compose up -d diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index d3cff12a4b32c4..00575a42c45b24 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -625,6 +625,8 @@ services: REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + SSRF_PROXY_ALLOW_PRIVATE_IPS: ${SSRF_PROXY_ALLOW_PRIVATE_IPS:-} + SSRF_PROXY_ALLOW_PRIVATE_DOMAINS: ${SSRF_PROXY_ALLOW_PRIVATE_DOMAINS:-} networks: - ssrf_proxy_network - default diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 170e1718565204..a93c29806ab9c2 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -215,6 +215,8 @@ services: REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + SSRF_PROXY_ALLOW_PRIVATE_IPS: ${SSRF_PROXY_ALLOW_PRIVATE_IPS:-} + SSRF_PROXY_ALLOW_PRIVATE_DOMAINS: ${SSRF_PROXY_ALLOW_PRIVATE_DOMAINS:-} ports: - "${EXPOSE_SSRF_PROXY_PORT:-3128}:${SSRF_HTTP_PORT:-3128}" - "${EXPOSE_SANDBOX_PORT:-8194}:${SANDBOX_PORT:-8194}" diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index da94f7f16c82e9..44cf15748def95 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -631,6 +631,8 @@ services: REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + SSRF_PROXY_ALLOW_PRIVATE_IPS: ${SSRF_PROXY_ALLOW_PRIVATE_IPS:-} + SSRF_PROXY_ALLOW_PRIVATE_DOMAINS: ${SSRF_PROXY_ALLOW_PRIVATE_DOMAINS:-} networks: - ssrf_proxy_network - default diff --git a/docker/envs/infrastructure/ssrf-proxy.env.example b/docker/envs/infrastructure/ssrf-proxy.env.example index 210a78249440ee..ac14687ff71d98 100644 --- a/docker/envs/infrastructure/ssrf-proxy.env.example +++ b/docker/envs/infrastructure/ssrf-proxy.env.example @@ -8,6 +8,8 @@ SSRF_HTTP_PORT=3128 SSRF_COREDUMP_DIR=/var/spool/squid SSRF_REVERSE_PROXY_PORT=8194 SSRF_SANDBOX_HOST=sandbox +SSRF_PROXY_ALLOW_PRIVATE_IPS= +SSRF_PROXY_ALLOW_PRIVATE_DOMAINS= SSRF_DEFAULT_TIME_OUT=5 SSRF_DEFAULT_CONNECT_TIME_OUT=5 SSRF_DEFAULT_READ_TIME_OUT=5 diff --git a/docker/envs/middleware.env.example b/docker/envs/middleware.env.example index 7b28a77fe3e9ad..f2cf84db7c914d 100644 --- a/docker/envs/middleware.env.example +++ b/docker/envs/middleware.env.example @@ -113,6 +113,8 @@ SSRF_HTTP_PORT=3128 SSRF_COREDUMP_DIR=/var/spool/squid SSRF_REVERSE_PROXY_PORT=8194 SSRF_SANDBOX_HOST=sandbox +SSRF_PROXY_ALLOW_PRIVATE_IPS= +SSRF_PROXY_ALLOW_PRIVATE_DOMAINS= # ------------------------------ # Environment Variables for weaviate Service @@ -240,4 +242,4 @@ LOGSTORE_DUAL_READ_ENABLED=true # Control flag for whether to write the `graph` field to LogStore. # If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; # otherwise write an empty {} instead. Defaults to writing the `graph` field. -LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true \ No newline at end of file +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true diff --git a/docker/ssrf_proxy/docker-entrypoint.sh b/docker/ssrf_proxy/docker-entrypoint.sh index 613897bb7db09c..54a32fe798f86b 100755 --- a/docker/ssrf_proxy/docker-entrypoint.sh +++ b/docker/ssrf_proxy/docker-entrypoint.sh @@ -26,6 +26,34 @@ tail -F /var/log/squid/error.log 2>/dev/null & tail -F /var/log/squid/store.log 2>/dev/null & tail -F /var/log/squid/cache.log 2>/dev/null & +ALLOW_PRIVATE_CONF=/etc/squid/dify_allow_private.conf + +write_optional_private_allowlist() { + local env_name="$1" + local acl_name="$2" + local acl_type="$3" + local raw_values="${!env_name:-}" + + raw_values="${raw_values//,/ }" + + if [ -z "${raw_values//[[:space:]]/}" ]; then + return + fi + + printf 'acl %s %s' "$acl_name" "$acl_type" >> "$ALLOW_PRIVATE_CONF" + for value in $raw_values; do + printf ' %s' "$value" >> "$ALLOW_PRIVATE_CONF" + done + printf '\nhttp_access allow client_localnet %s\n' "$acl_name" >> "$ALLOW_PRIVATE_CONF" +} + +{ + echo "# Generated by docker-entrypoint.sh." + echo "# Allows selected private targets before the default private-network deny rule." +} > "$ALLOW_PRIVATE_CONF" +write_optional_private_allowlist "SSRF_PROXY_ALLOW_PRIVATE_IPS" "dify_allowed_private_networks" "dst" +write_optional_private_allowlist "SSRF_PROXY_ALLOW_PRIVATE_DOMAINS" "dify_allowed_private_domains" "dstdomain" + # Replace environment variables in the template and output to the squid.conf echo "[ENTRYPOINT] replacing environment variables in the template" awk '{ diff --git a/docker/ssrf_proxy/squid.conf.template b/docker/ssrf_proxy/squid.conf.template index fbe9ebc448b0e4..e308f85f2f674a 100644 --- a/docker/ssrf_proxy/squid.conf.template +++ b/docker/ssrf_proxy/squid.conf.template @@ -1,11 +1,23 @@ -acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) -acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) -acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) -acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines -acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) -acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) -acl localnet src fc00::/7 # RFC 4193 local private network range -acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl client_localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl client_localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl client_localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl client_localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl client_localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl client_localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl client_localnet src fc00::/7 # RFC 4193 local private network range +acl client_localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl to_private_networks dst 0.0.0.0/32 +acl to_private_networks dst 10.0.0.0/8 +acl to_private_networks dst 100.64.0.0/10 +acl to_private_networks dst 127.0.0.0/8 +acl to_private_networks dst 169.254.0.0/16 +acl to_private_networks dst 172.16.0.0/12 +acl to_private_networks dst 192.168.0.0/16 +acl to_private_networks dst 224.0.0.0/4 +acl to_private_networks dst 240.0.0.0/4 +acl to_private_networks dst ::1/128 +acl to_private_networks dst fc00::/7 +acl to_private_networks dst fe80::/10 acl SSL_ports port 443 # acl SSL_ports port 1025-65535 # Enable the configuration to resolve this issue: https://github.com/langgenius/dify/issues/12792 acl Safe_ports port 80 # http @@ -19,19 +31,27 @@ acl Safe_ports port 488 # gss-http acl Safe_ports port 591 # filemaker acl Safe_ports port 777 # multiling http acl CONNECT method CONNECT +acl sandbox_reverse_proxy_port localport ${REVERSE_PROXY_PORT} acl allowed_domains dstdomain .marketplace.dify.ai -http_access allow allowed_domains + +http_port ${HTTP_PORT} +http_port ${REVERSE_PROXY_PORT} accel vhost +cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver + http_access deny !Safe_ports http_access deny CONNECT !SSL_ports http_access allow localhost manager http_access deny manager +http_access allow sandbox_reverse_proxy_port +include /etc/squid/dify_allow_private.conf +http_access deny to_private_networks +http_access allow allowed_domains +http_access allow client_localnet http_access allow localhost -include /etc/squid/conf.d/*.conf http_access deny all tcp_outgoing_address 0.0.0.0 ################################## Proxy Server ################################ -http_port ${HTTP_PORT} coredump_dir ${COREDUMP_DIR} refresh_pattern ^ftp: 1440 20% 10080 refresh_pattern ^gopher: 1440 0% 1440 @@ -48,10 +68,6 @@ refresh_pattern . 0 20% 4320 # cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default ################################## Reverse Proxy To Sandbox ################################ -http_port ${REVERSE_PROXY_PORT} accel vhost -cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver -acl src_all src all -http_access allow src_all # Unless the option's size is increased, an error will occur when uploading more than two files. client_request_buffer_max_size 100 MB @@ -103,4 +119,3 @@ access_log daemon:/var/log/squid/access.log dify_log # Access log to track concurrent requests and timeouts logfile_rotate 10 - diff --git a/docker/ssrf_proxy/test_ssrf_proxy_config.sh b/docker/ssrf_proxy/test_ssrf_proxy_config.sh new file mode 100755 index 00000000000000..80a546ca01cbc2 --- /dev/null +++ b/docker/ssrf_proxy/test_ssrf_proxy_config.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +IMAGE="${SSRF_PROXY_TEST_IMAGE:-ubuntu/squid:latest}" +CLIENT_IMAGE="${SSRF_PROXY_TEST_CLIENT_IMAGE:-busybox:latest}" +CONTAINER_NAME="${SSRF_PROXY_TEST_CONTAINER:-dify-ssrf-proxy-test-$$}" +SANDBOX_CONTAINER_NAME="${CONTAINER_NAME}-sandbox" +NETWORK_NAME="${SSRF_PROXY_TEST_NETWORK:-dify-ssrf-proxy-test-$$}" +RUN_PUBLIC_CHECK="${SSRF_PROXY_TEST_PUBLIC_CHECK:-true}" + +cleanup() { + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker rm -f "$SANDBOX_CONTAINER_NAME" >/dev/null 2>&1 || true + docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true +} + +http_code_for() { + local proxy_url="$1" + local target_url="$2" + local output + + output="$( + docker run \ + --rm \ + --network "$NETWORK_NAME" \ + --env "http_proxy=$proxy_url" \ + --env "https_proxy=$proxy_url" \ + "$CLIENT_IMAGE" \ + wget -S -O /dev/null -T 10 "$target_url" 2>&1 || true + )" + + printf '%s\n' "$output" | awk '$1 ~ /^HTTP\// { code = $2 } END { print code }' +} + +direct_http_code_for() { + local target_url="$1" + local output + + output="$( + docker run \ + --rm \ + --network "$NETWORK_NAME" \ + "$CLIENT_IMAGE" \ + wget -S -O /dev/null -T 10 "$target_url" 2>&1 || true + )" + + printf '%s\n' "$output" | awk '$1 ~ /^HTTP\// { code = $2 } END { print code }' +} + +assert_private_target_blocked() { + local proxy_url="$1" + local target_url="$2" + local status_code + + status_code="$(http_code_for "$proxy_url" "$target_url")" + if [[ "$status_code" != "403" ]]; then + echo "Expected $target_url to be blocked with HTTP 403, got ${status_code:-no response}." + docker logs "$CONTAINER_NAME" >&2 || true + exit 1 + fi +} + +assert_public_target_allowed() { + local proxy_url="$1" + local target_url="$2" + local status_code + + status_code="$(http_code_for "$proxy_url" "$target_url")" + if [[ ! "$status_code" =~ ^(2|3)[0-9][0-9]$ ]]; then + echo "Expected $target_url to remain reachable, got ${status_code:-no response}." + docker logs "$CONTAINER_NAME" >&2 || true + exit 1 + fi +} + +assert_reverse_proxy_allowed() { + local target_url="$1" + local status_code + + status_code="$(direct_http_code_for "$target_url")" + if [[ ! "$status_code" =~ ^2[0-9][0-9]$ ]]; then + echo "Expected sandbox reverse proxy $target_url to remain reachable, got ${status_code:-no response}." + docker logs "$CONTAINER_NAME" >&2 || true + docker logs "$SANDBOX_CONTAINER_NAME" >&2 || true + exit 1 + fi +} + +trap cleanup EXIT +cleanup +docker network create "$NETWORK_NAME" >/dev/null + +docker run \ + --detach \ + --name "$SANDBOX_CONTAINER_NAME" \ + --network "$NETWORK_NAME" \ + --network-alias sandbox \ + "$CLIENT_IMAGE" \ + sh -c "mkdir -p /www && echo ok > /www/index.html && httpd -f -p 8194 -h /www" \ + >/dev/null + +docker run \ + --detach \ + --name "$CONTAINER_NAME" \ + --entrypoint sh \ + --network "$NETWORK_NAME" \ + --volume "$ROOT_DIR/docker/ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template:ro" \ + --volume "$ROOT_DIR/docker/ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh:ro" \ + --env HTTP_PORT=3128 \ + --env COREDUMP_DIR=/var/spool/squid \ + --env REVERSE_PROXY_PORT=8194 \ + --env SANDBOX_HOST=sandbox \ + --env SANDBOX_PORT=8194 \ + --env "SSRF_PROXY_ALLOW_PRIVATE_IPS=${SSRF_PROXY_ALLOW_PRIVATE_IPS:-}" \ + --env "SSRF_PROXY_ALLOW_PRIVATE_DOMAINS=${SSRF_PROXY_ALLOW_PRIVATE_DOMAINS:-}" \ + "$IMAGE" \ + -c "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" \ + >/dev/null + +proxy_url="http://$CONTAINER_NAME:3128" +for _ in {1..30}; do + probe_status="$(http_code_for "$proxy_url" "http://127.0.0.1:80/")" + if [[ -n "$probe_status" ]]; then + break + fi + sleep 1 +done + +if [[ -z "${probe_status:-}" ]]; then + echo "Squid proxy did not respond to probes." + docker logs "$CONTAINER_NAME" >&2 || true + exit 1 +fi + +assert_private_target_blocked "$proxy_url" "http://127.0.0.1:80/" +assert_private_target_blocked "$proxy_url" "http://169.254.169.254/latest/meta-data/" + +if [[ "$RUN_PUBLIC_CHECK" == "true" ]]; then + assert_public_target_allowed "$proxy_url" "http://example.com/" +fi + +assert_reverse_proxy_allowed "http://$CONTAINER_NAME:8194/"