diff --git a/byconity/benchmark.sh b/byconity/benchmark.sh index 0450372b8a..b71ae2462e 100755 --- a/byconity/benchmark.sh +++ b/byconity/benchmark.sh @@ -7,4 +7,10 @@ export BENCH_DURABLE=yes # dependency, so the worst-case cold start is several minutes; the # lib's 300s default has timed out before server is up. export BENCH_CHECK_TIMEOUT=1200 +# After firecracker snapshot+restore the cluster's +# internal connections (brpc/gossip) are stale; ./start's +# shallow health probe doesn't notice and short-circuits. +# Tell the playground agent to ./stop the cluster before +# ./start so the next bring-up is from a clean state. +export PLAYGROUND_RESTART_AFTER_RESTORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/cedardb-parquet/start b/cedardb-parquet/start index 981f23f221..f12fcd366d 100755 --- a/cedardb-parquet/start +++ b/cedardb-parquet/start @@ -28,10 +28,15 @@ if ! sudo docker run -d --rm -p 5432:5432 \ exit 1 fi -for _ in $(seq 1 60); do +# First-boot initdb inside the container takes well over a minute +# (observed ~90-120 s of "Fixing permissions"/"Setting up database +# directory" before postgres actually listens). Give it 10 min — +# pg_isready exits fast once the daemon is up, so this only +# matters in the failure path. +for _ in $(seq 1 600); do pg_isready -h localhost --dbname postgres -U postgres >/dev/null 2>&1 && exit 0 sleep 1 done -echo "cedardb did not become ready in 60 s; container logs:" >&2 +echo "cedardb did not become ready in 600 s; container logs:" >&2 sudo docker logs cedardb 2>&1 | tail -40 >&2 || true exit 1 diff --git a/cedardb/start b/cedardb/start index b6c3bbfe07..6bd35d176f 100755 --- a/cedardb/start +++ b/cedardb/start @@ -30,10 +30,15 @@ if ! sudo docker run -d --rm -p 5432:5432 \ exit 1 fi -for _ in $(seq 1 60); do +# First-boot initdb inside the container can run for well over a +# minute (observed ~90-120 s of "Fixing permissions"/"Setting up +# database directory" before postgres actually listens). Older +# 60 s budget bailed during that phase. Give it 5 min — pg_isready +# exits fast once the daemon is up so this only matters on failure. +for _ in $(seq 1 600); do pg_isready -h localhost --dbname postgres -U postgres >/dev/null 2>&1 && exit 0 sleep 1 done -echo "cedardb did not become ready in 60 s; container logs:" >&2 +echo "cedardb did not become ready in 600 s; container logs:" >&2 sudo docker logs cedardb 2>&1 | tail -40 >&2 || true exit 1 diff --git a/chdb-dataframe/benchmark.sh b/chdb-dataframe/benchmark.sh index 6bf667e4f0..4148581913 100755 --- a/chdb-dataframe/benchmark.sh +++ b/chdb-dataframe/benchmark.sh @@ -2,4 +2,9 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=no +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/clickhouse-web/create.sql b/clickhouse-web/create.sql index 4e687ef61f..3ec2451dc7 100644 --- a/clickhouse-web/create.sql +++ b/clickhouse-web/create.sql @@ -108,5 +108,5 @@ ATTACH TABLE hits UUID 'c449dfbf-ba06-4d13-abec-8396559eb955' PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID) ) ENGINE = MergeTree -SETTINGS disk = disk(type = cache, path = '/dev/shm/clickhouse/', max_size_ratio_to_total_space = 0.9, +SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/caches/web/', max_size_ratio_to_total_space = 0.9, disk = disk(type = web, endpoint = 'https://clickhouse-public-datasets.s3.amazonaws.com/web/')); diff --git a/clickhouse-web/install b/clickhouse-web/install index eb23629536..75715b33c6 100755 --- a/clickhouse-web/install +++ b/clickhouse-web/install @@ -10,6 +10,18 @@ if [ ! -x /usr/bin/clickhouse ]; then sudo ./clickhouse install --noninteractive fi -# Cache directory used by the web disk. -sudo mkdir -p /dev/shm/clickhouse -sudo chown clickhouse:clickhouse /dev/shm/clickhouse +# Cache directory used by the web disk. ClickHouse rejects any +# filesystem-cache path outside /var/lib/clickhouse/caches/ with +# BAD_ARGUMENTS at CREATE TABLE time, but we still want the actual +# bytes to live in tmpfs (/dev/shm) for the speed: cold queries +# pull ~1 GB on first run and tmpfs avoids touching the host SSD. +# +# Newer ClickHouse versions canonicalise the path before the policy +# check, so the older symlink trick (caches/web → /dev/shm/...) is +# rejected with BAD_ARGUMENTS. Bind-mount tmpfs at the +# policy-acceptable path instead — to CH the cache dir *is* +# /var/lib/clickhouse/caches/web with no symlink to resolve. +sudo mkdir -p /dev/shm/clickhouse /var/lib/clickhouse/caches/web +sudo chown clickhouse:clickhouse /dev/shm/clickhouse /var/lib/clickhouse/caches/web +sudo mount --bind /dev/shm/clickhouse /var/lib/clickhouse/caches/web +sudo chown clickhouse:clickhouse /var/lib/clickhouse/caches/web diff --git a/daft-parquet-partitioned/benchmark.sh b/daft-parquet-partitioned/benchmark.sh index 1495c0bf62..024c58fe16 100755 --- a/daft-parquet-partitioned/benchmark.sh +++ b/daft-parquet-partitioned/benchmark.sh @@ -2,4 +2,9 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=no +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/daft-parquet/.preserve-state b/daft-parquet/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/daft-parquet/benchmark.sh b/daft-parquet/benchmark.sh index 6bf667e4f0..4148581913 100755 --- a/daft-parquet/benchmark.sh +++ b/daft-parquet/benchmark.sh @@ -2,4 +2,9 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=no +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/druid/load b/druid/load index b03e0b59ad..0f3a8f6e8e 100755 --- a/druid/load +++ b/druid/load @@ -11,8 +11,15 @@ DRUID_DIR="apache-druid-${VERSION}" # datasource). "./${DRUID_DIR}/bin/post-index-task" --file ingest.json --url http://localhost:8081 || true -# Wait until the hits datasource is queryable. -for _ in $(seq 1 600); do +# Wait until the hits datasource is queryable. Druid's index task can +# legitimately take hours on a 16 GiB VM; budget 4 h here, and fail +# loudly if hits still isn't queryable so the agent doesn't take a +# snapshot of a half-ingested datasource (which would otherwise look +# "snapshotted" but every query returns +# druidException ... Object 'hits' not found +# at runtime). +cnt="" +for _ in $(seq 1 2880); do # 2880 * 5s = 4 h cnt=$(curl -sf -XPOST -H'Content-Type: application/json' \ http://localhost:8888/druid/v2/sql/ \ -d '{"query": "SELECT COUNT(*) FROM hits"}' 2>/dev/null \ @@ -22,6 +29,13 @@ for _ in $(seq 1 600); do fi sleep 5 done +if [ -z "$cnt" ] || [ "$cnt" -le 0 ]; then + echo "druid: hits datasource still not queryable after 4 h; ingestion" >&2 + echo "did not finish. Dumping recent task list for diagnosis:" >&2 + curl -sS http://localhost:8081/druid/indexer/v1/tasks 2>&1 | head -c 2000 >&2 + exit 1 +fi +echo "druid: hits has $cnt rows after ingestion" rm -f hits.tsv sync diff --git a/duckdb-dataframe/.preserve-state b/duckdb-dataframe/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/duckdb-dataframe/benchmark.sh b/duckdb-dataframe/benchmark.sh index 6bf667e4f0..4148581913 100755 --- a/duckdb-dataframe/benchmark.sh +++ b/duckdb-dataframe/benchmark.sh @@ -2,4 +2,9 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=no +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/firebolt-parquet-partitioned/benchmark.sh b/firebolt-parquet-partitioned/benchmark.sh index 0e6a62ae64..dbb9072c56 100755 --- a/firebolt-parquet-partitioned/benchmark.sh +++ b/firebolt-parquet-partitioned/benchmark.sh @@ -1,49 +1,6 @@ #!/bin/bash - -# Download the partitioned hits parquet files -echo "Downloading dataset..." -rm -rf data -../lib/download-hits-parquet-partitioned data - -# Start the container -sudo apt-get install -y docker.io jq -sudo docker run -dit --name firebolt-core --rm \ - --ulimit memlock=8589934592:8589934592 \ - --security-opt seccomp=unconfined \ - -p 127.0.0.1:3473:3473 \ - -v /firebolt-core/volume \ - -v ./data/:/firebolt-core/clickbench \ - ghcr.io/firebolt-db/firebolt-core:preview-rc - -# See firebolt/benchmark.sh — the old curl-and-break pattern accepted the -# "Cluster not yet healthy" JSON error body as success. -for _ in {1..600} -do - if curl -sS "http://localhost:3473/" \ - --data-binary "SELECT 'Firebolt is ready';" 2>/dev/null \ - | grep -q "Firebolt is ready"; then - break - fi - sleep 1 -done - -# Create the database and external table -echo "Creating external table..." -curl -sS "http://localhost:3473/?enable_multi_query_requests=true" --data-binary "DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;" -curl -sS "http://localhost:3473/?database=clickbench&enable_multi_query_requests=true" --data-binary @create.sql - -# Print statistics -DATA_SIZE=$(du -bcs data/hits_*.parquet 2>/dev/null | grep total | awk '{print $1}') -if [ -z "$DATA_SIZE" ]; then - DATA_SIZE=$(du -cs data/hits_*.parquet | grep total | awk '{print $1}') -fi -echo "Load time: 0" -echo "Data size: $DATA_SIZE" - -# Run the benchmark -echo "Running the benchmark..." -./run.sh - -# Stop the container and remove the data -sudo docker container stop firebolt-core -rm -rf data +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +export BENCH_DURABLE=no +export BENCH_RESTARTABLE=no +exec ../lib/benchmark-common.sh diff --git a/firebolt-parquet-partitioned/check b/firebolt-parquet-partitioned/check new file mode 100755 index 0000000000..862722f602 --- /dev/null +++ b/firebolt-parquet-partitioned/check @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +# Firebolt-core's HTTP port answers immediately but may return a +# cluster-not-ready JSON error at HTTP 200. Test for an actual result. +curl -sSf --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1' diff --git a/firebolt-parquet-partitioned/data-size b/firebolt-parquet-partitioned/data-size new file mode 100755 index 0000000000..b5fe999ff8 --- /dev/null +++ b/firebolt-parquet-partitioned/data-size @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Firebolt-core writes its database state under /firebolt-core/volume +# inside the container, which we bind-mount to ./fb-volume on the host. +du -bcs fb-volume 2>/dev/null | awk '/total$/ { print $1 }' diff --git a/firebolt-parquet-partitioned/install b/firebolt-parquet-partitioned/install new file mode 100755 index 0000000000..38799727d9 --- /dev/null +++ b/firebolt-parquet-partitioned/install @@ -0,0 +1,6 @@ +#!/bin/bash +set -eu + +sudo apt-get update -y +sudo apt-get install -y docker.io jq +sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc diff --git a/firebolt-parquet-partitioned/load b/firebolt-parquet-partitioned/load new file mode 100755 index 0000000000..e309c8968f --- /dev/null +++ b/firebolt-parquet-partitioned/load @@ -0,0 +1,20 @@ +#!/bin/bash +set -eu + +# Partitioned-parquet variant: stage hits_*.parquet under ./data so +# the container sees them at /firebolt-core/clickbench/*.parquet; +# create.sql declares an external table with FROM PATTERN that +# matches the glob. +mkdir -p data +shopt -s nullglob +for f in hits_*.parquet; do + mv -f "$f" "data/$f" +done +shopt -u nullglob + +curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \ + --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;' +curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \ + --data-binary @create.sql + +sync diff --git a/firebolt-parquet-partitioned/query b/firebolt-parquet-partitioned/query new file mode 100755 index 0000000000..910591e6b8 --- /dev/null +++ b/firebolt-parquet-partitioned/query @@ -0,0 +1,28 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it against the firebolt-core +# container via /?database=clickbench. +# Stdout: query result (firebolt's JSON_Compact format). +# Stderr: query runtime in fractional seconds on the last line, +# pulled from the response's `.statistics.elapsed`. +# Exit non-zero on error. +set -e + +query=$(cat) + +# Result + sub-result caches off so timings are real; output_format +# matches what firebolt's run.sh uses for the public benchmark. +PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact' + +resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \ + --data-binary "$query") + +# Firebolt returns a JSON object whether the query succeeded or not. +# A failed query has an "errors" key; a successful one carries +# "data" + "statistics". +if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then + printf '%s\n' "$resp" >&2 + exit 1 +fi + +printf '%s\n' "$resp" +printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2 diff --git a/firebolt-parquet-partitioned/run.sh b/firebolt-parquet-partitioned/run.sh deleted file mode 100755 index 9b810c99ea..0000000000 --- a/firebolt-parquet-partitioned/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Disable the result and subresult caches. -QUERY_PARAMS="enable_result_cache=false&enable_subresult_cache=false&output_format=JSON_Compact" - -cat queries.sql | while read -r query; do - # Firebolt is a database with local on-disk storage: drop the page cache before the first run of each query. - sync - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - # Run the query three times. - # Extract the elapsed time from the response's statistics. - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n "[${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n ",${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo ",${ELAPSED}]," -done diff --git a/firebolt-parquet-partitioned/start b/firebolt-parquet-partitioned/start new file mode 100755 index 0000000000..12fa615cd7 --- /dev/null +++ b/firebolt-parquet-partitioned/start @@ -0,0 +1,76 @@ +#!/bin/bash +set -eu + +# Idempotent: if firebolt-core already answers SELECT 1, do nothing. +if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'; then + exit 0 +fi + +mkdir -p data fb-volume +# firebolt-core runs as UID/GID 1111 inside the container and refuses +# to start if its data dir is not writeable by that uid (the engine +# self-checks and aborts with "directory ... is not readable or +# writeable by the Firebolt Core process"). Set the host-side +# ownership accordingly so the bind-mounted dir is usable. +sudo chown 1111:1111 fb-volume + +# If the container exists (stopped from a prior agent pre-snapshot +# cycle), just start it back — the data lives on the bind-mounted +# fb-volume below, so the previously-created `clickbench` database +# is still there. Otherwise create the container fresh. +if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then + sudo docker start firebolt-core >/dev/null +else + # `firebolt-core` is the public self-hosted image. Container needs + # memlock 8 GiB and seccomp unconfined per upstream's run docs. + # /firebolt-core/clickbench: parquet source (read at load time). + # /firebolt-core/volume: engine data directory (must persist + # across the agent's pre-snapshot + # stop+start cycle or the snapshot + # ships an empty DB). + # The agent stages partitioned parquet at $PWD as symlinks pointing + # at /opt/clickbench/datasets_ro/hits_partitioned/hits_N.parquet + # (an absolute host-VM path). `./load` then `mv`s those symlinks + # into data/, but inside the container the absolute target is + # unreachable and the symlinks dangle — load reports success but + # the external table sees zero files and every query returns + # rows_read=0. Bind-mount the dataset disk into the container at + # the same absolute path so the symlinks resolve. + sudo docker run -dit --name firebolt-core \ + --ulimit memlock=8589934592:8589934592 \ + --security-opt seccomp=unconfined \ + -p 127.0.0.1:3473:3473 \ + -v "$(pwd)/fb-volume:/firebolt-core/volume" \ + -v "$(pwd)/data:/firebolt-core/clickbench" \ + -v "/opt/clickbench/datasets_ro:/opt/clickbench/datasets_ro:ro" \ + ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null +fi + +# Wait for the cluster to be "actually" ready. firebolt-core's HTTP +# port comes up immediately but returns +# {"errors":[{"description":"Cluster not yet healthy: ..."}]} +# at HTTP 200 until the engine threads have warmed; bench against a +# sentinel string instead of HTTP status to avoid that trap. +for _ in $(seq 1 600); do + if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary "SELECT 'firebolt-ready';" 2>/dev/null \ + | grep -q 'firebolt-ready'; then + exit 0 + fi + sleep 1 +done +{ + echo "firebolt-core did not become healthy in 10 min" + echo "=== docker ps -a ===" + sudo docker ps -a 2>&1 + echo "=== docker inspect firebolt-core (state) ===" + sudo docker inspect firebolt-core --format '{{json .State}}' 2>&1 + echo "=== docker logs firebolt-core --tail 50 ===" + sudo docker logs firebolt-core --tail 50 2>&1 + echo "=== curl http://localhost:3473/ ===" + curl -sS --max-time 3 'http://localhost:3473/' --data-binary 'SELECT 1' 2>&1 + echo "=== ss listeners ===" + sudo ss -lntp 2>&1 | head -20 +} >&2 +exit 1 diff --git a/firebolt-parquet-partitioned/stop b/firebolt-parquet-partitioned/stop new file mode 100755 index 0000000000..ac1834f7d6 --- /dev/null +++ b/firebolt-parquet-partitioned/stop @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Plain stop — leave the container in place so its bind-mounted +# fb-volume keeps the loaded database for the next ./start. The +# container is removed and the volume re-initialised only on +# explicit re-provision. +sudo docker container stop firebolt-core >/dev/null 2>&1 || true diff --git a/firebolt-parquet/benchmark.sh b/firebolt-parquet/benchmark.sh index 737a3ca865..3a332d30db 100755 --- a/firebolt-parquet/benchmark.sh +++ b/firebolt-parquet/benchmark.sh @@ -1,46 +1,6 @@ #!/bin/bash - -# Download the hits.parquet file -echo "Downloading dataset..." -rm -rf data -../lib/download-hits-parquet-single data - -# Start the container -sudo apt-get install -y docker.io jq -sudo docker run -dit --name firebolt-core --rm \ - --ulimit memlock=8589934592:8589934592 \ - --security-opt seccomp=unconfined \ - -p 127.0.0.1:3473:3473 \ - -v /firebolt-core/volume \ - -v ./data/:/firebolt-core/clickbench \ - ghcr.io/firebolt-db/firebolt-core:preview-rc - -# See firebolt/benchmark.sh — the old curl-and-break pattern accepted the -# "Cluster not yet healthy" JSON error body as success. -for _ in {1..600} -do - if curl -sS "http://localhost:3473/" \ - --data-binary "SELECT 'Firebolt is ready';" 2>/dev/null \ - | grep -q "Firebolt is ready"; then - break - fi - sleep 1 -done - -# Create the database and external table -echo "Creating external table..." -curl -sS "http://localhost:3473/?enable_multi_query_requests=true" --data-binary "DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;" -curl -sS "http://localhost:3473/?database=clickbench&enable_multi_query_requests=true" --data-binary @create.sql - -# Print statistics -DATA_SIZE=$(stat -c%s data/hits.parquet 2>/dev/null || stat -f%z data/hits.parquet) -echo "Load time: 0" -echo "Data size: $DATA_SIZE" - -# Run the benchmark -echo "Running the benchmark..." -./run.sh - -# Stop the container and remove the data -sudo docker container stop firebolt-core -rm -rf data +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_DURABLE=no +export BENCH_RESTARTABLE=no +exec ../lib/benchmark-common.sh diff --git a/firebolt-parquet/check b/firebolt-parquet/check new file mode 100755 index 0000000000..862722f602 --- /dev/null +++ b/firebolt-parquet/check @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +# Firebolt-core's HTTP port answers immediately but may return a +# cluster-not-ready JSON error at HTTP 200. Test for an actual result. +curl -sSf --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1' diff --git a/firebolt-parquet/data-size b/firebolt-parquet/data-size new file mode 100755 index 0000000000..b5fe999ff8 --- /dev/null +++ b/firebolt-parquet/data-size @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Firebolt-core writes its database state under /firebolt-core/volume +# inside the container, which we bind-mount to ./fb-volume on the host. +du -bcs fb-volume 2>/dev/null | awk '/total$/ { print $1 }' diff --git a/firebolt-parquet/install b/firebolt-parquet/install new file mode 100755 index 0000000000..38799727d9 --- /dev/null +++ b/firebolt-parquet/install @@ -0,0 +1,6 @@ +#!/bin/bash +set -eu + +sudo apt-get update -y +sudo apt-get install -y docker.io jq +sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc diff --git a/firebolt-parquet/load b/firebolt-parquet/load new file mode 100755 index 0000000000..8d3886ee9d --- /dev/null +++ b/firebolt-parquet/load @@ -0,0 +1,17 @@ +#!/bin/bash +set -eu + +# Parquet variant: data stays in ./data (mounted as +# /firebolt-core/clickbench in the container), create.sql declares +# an external table that reads it on every query. +mkdir -p data +if [ -f hits.parquet ]; then + mv -f hits.parquet data/hits.parquet +fi + +curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \ + --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;' +curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \ + --data-binary @create.sql + +sync diff --git a/firebolt-parquet/query b/firebolt-parquet/query new file mode 100755 index 0000000000..910591e6b8 --- /dev/null +++ b/firebolt-parquet/query @@ -0,0 +1,28 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it against the firebolt-core +# container via /?database=clickbench. +# Stdout: query result (firebolt's JSON_Compact format). +# Stderr: query runtime in fractional seconds on the last line, +# pulled from the response's `.statistics.elapsed`. +# Exit non-zero on error. +set -e + +query=$(cat) + +# Result + sub-result caches off so timings are real; output_format +# matches what firebolt's run.sh uses for the public benchmark. +PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact' + +resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \ + --data-binary "$query") + +# Firebolt returns a JSON object whether the query succeeded or not. +# A failed query has an "errors" key; a successful one carries +# "data" + "statistics". +if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then + printf '%s\n' "$resp" >&2 + exit 1 +fi + +printf '%s\n' "$resp" +printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2 diff --git a/firebolt-parquet/run.sh b/firebolt-parquet/run.sh deleted file mode 100755 index 9b810c99ea..0000000000 --- a/firebolt-parquet/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Disable the result and subresult caches. -QUERY_PARAMS="enable_result_cache=false&enable_subresult_cache=false&output_format=JSON_Compact" - -cat queries.sql | while read -r query; do - # Firebolt is a database with local on-disk storage: drop the page cache before the first run of each query. - sync - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - # Run the query three times. - # Extract the elapsed time from the response's statistics. - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n "[${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n ",${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo ",${ELAPSED}]," -done diff --git a/firebolt-parquet/start b/firebolt-parquet/start new file mode 100755 index 0000000000..652146394e --- /dev/null +++ b/firebolt-parquet/start @@ -0,0 +1,67 @@ +#!/bin/bash +set -eu + +# Idempotent: if firebolt-core already answers SELECT 1, do nothing. +if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'; then + exit 0 +fi + +mkdir -p data fb-volume +# firebolt-core runs as UID/GID 1111 inside the container and refuses +# to start if its data dir is not writeable by that uid (the engine +# self-checks and aborts with "directory ... is not readable or +# writeable by the Firebolt Core process"). Set the host-side +# ownership accordingly so the bind-mounted dir is usable. +sudo chown 1111:1111 fb-volume + +# If the container exists (stopped from a prior agent pre-snapshot +# cycle), just start it back — the data lives on the bind-mounted +# fb-volume below, so the previously-created `clickbench` database +# is still there. Otherwise create the container fresh. +if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then + sudo docker start firebolt-core >/dev/null +else + # `firebolt-core` is the public self-hosted image. Container needs + # memlock 8 GiB and seccomp unconfined per upstream's run docs. + # /firebolt-core/clickbench: parquet source (read at load time). + # /firebolt-core/volume: engine data directory (must persist + # across the agent's pre-snapshot + # stop+start cycle or the snapshot + # ships an empty DB). + sudo docker run -dit --name firebolt-core \ + --ulimit memlock=8589934592:8589934592 \ + --security-opt seccomp=unconfined \ + -p 127.0.0.1:3473:3473 \ + -v "$(pwd)/fb-volume:/firebolt-core/volume" \ + -v "$(pwd)/data:/firebolt-core/clickbench" \ + ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null +fi + +# Wait for the cluster to be "actually" ready. firebolt-core's HTTP +# port comes up immediately but returns +# {"errors":[{"description":"Cluster not yet healthy: ..."}]} +# at HTTP 200 until the engine threads have warmed; bench against a +# sentinel string instead of HTTP status to avoid that trap. +for _ in $(seq 1 600); do + if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary "SELECT 'firebolt-ready';" 2>/dev/null \ + | grep -q 'firebolt-ready'; then + exit 0 + fi + sleep 1 +done +{ + echo "firebolt-core did not become healthy in 10 min" + echo "=== docker ps -a ===" + sudo docker ps -a 2>&1 + echo "=== docker inspect firebolt-core (state) ===" + sudo docker inspect firebolt-core --format '{{json .State}}' 2>&1 + echo "=== docker logs firebolt-core --tail 50 ===" + sudo docker logs firebolt-core --tail 50 2>&1 + echo "=== curl http://localhost:3473/ ===" + curl -sS --max-time 3 'http://localhost:3473/' --data-binary 'SELECT 1' 2>&1 + echo "=== ss listeners ===" + sudo ss -lntp 2>&1 | head -20 +} >&2 +exit 1 diff --git a/firebolt-parquet/stop b/firebolt-parquet/stop new file mode 100755 index 0000000000..ac1834f7d6 --- /dev/null +++ b/firebolt-parquet/stop @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Plain stop — leave the container in place so its bind-mounted +# fb-volume keeps the loaded database for the next ./start. The +# container is removed and the volume re-initialised only on +# explicit re-provision. +sudo docker container stop firebolt-core >/dev/null 2>&1 || true diff --git a/firebolt/benchmark.sh b/firebolt/benchmark.sh index f27bc92f71..617422ddc2 100755 --- a/firebolt/benchmark.sh +++ b/firebolt/benchmark.sh @@ -1,63 +1,6 @@ #!/bin/bash - -# Download the hits.parquet file -echo "Downloading dataset..." -rm -rf data -../lib/download-hits-parquet-single data - -# Start the container -sudo apt-get install -y docker.io jq -sudo docker run -dit --name firebolt-core --rm \ - --ulimit memlock=8589934592:8589934592 \ - --security-opt seccomp=unconfined \ - -p 127.0.0.1:3473:3473 \ - -v /firebolt-core/volume \ - -v ./data/:/firebolt-core/clickbench \ - ghcr.io/firebolt-db/firebolt-core:preview-rc - -# Wait until Firebolt is ready. The old loop just did -# curl -s ... > /dev/null && break -# which treated any HTTP response as success, including the JSON error -# body -# {"errors":[{"description":"Cluster not yet healthy: ..."}]} -# that Firebolt returns at HTTP 200 while the container is still -# warming up. The loop exited on the first reply, the next -# CREATE TABLE / queries all hit the same "Cluster not yet healthy" -# error, and every query got recorded as "elapsed":0.0 — sink.parser -# then rejected the run for having no timing > 0.1 s, which is why -# Firebolt stopped showing up in sink.results after 2026-02-21 -# despite the bench completing 43/43 each time. -for _ in {1..600} -do - if curl -sS "http://localhost:3473/" \ - --data-binary "SELECT 'Firebolt is ready';" 2>/dev/null \ - | grep -q "Firebolt is ready"; then - break - fi - sleep 1 -done - -# Ingest the data -echo "Ingesting the data..." -curl -s "http://localhost:3473/?enable_multi_query_requests=true" --data-binary "DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;" -LOAD_TIME=$(curl -w "%{time_total}\n" -s "http://localhost:3473/?database=clickbench&enable_multi_query_requests=true" --data-binary @create.sql) - -# Print statistics -COMPRESSED_SIZE=$(curl -s "http://localhost:3473/?database=clickbench&output_format=JSON_Compact" --data-binary "SELECT compressed_bytes FROM information_schema.tables WHERE table_name = 'hits';" | jq '.data[0][0] | tonumber') -UNCOMPRESSED_SIZE=$(curl -s "http://localhost:3473/?database=clickbench&output_format=JSON_Compact" --data-binary "SELECT uncompressed_bytes FROM information_schema.tables WHERE table_name = 'hits';" | jq '.data[0][0] | tonumber') -echo "Load time: $LOAD_TIME" -echo "Data size: $COMPRESSED_SIZE" -echo "Uncompressed data size: $UNCOMPRESSED_SIZE bytes" - -if [ "$1" != "" ] && [ "$1" != "scan-cache" ]; then - echo "Error: command line argument must be one of {'', 'scan-cache'}" - exit 1 -fi - -# Run the benchmark -echo "Running the benchmark..." -./run.sh "$1" - -# Stop the container and remove the data -sudo docker container stop firebolt-core -rm -rf data +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_DURABLE=yes +export BENCH_RESTARTABLE=no +exec ../lib/benchmark-common.sh diff --git a/firebolt/check b/firebolt/check new file mode 100755 index 0000000000..862722f602 --- /dev/null +++ b/firebolt/check @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +# Firebolt-core's HTTP port answers immediately but may return a +# cluster-not-ready JSON error at HTTP 200. Test for an actual result. +curl -sSf --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1' diff --git a/firebolt/data-size b/firebolt/data-size new file mode 100755 index 0000000000..b5fe999ff8 --- /dev/null +++ b/firebolt/data-size @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Firebolt-core writes its database state under /firebolt-core/volume +# inside the container, which we bind-mount to ./fb-volume on the host. +du -bcs fb-volume 2>/dev/null | awk '/total$/ { print $1 }' diff --git a/firebolt/install b/firebolt/install new file mode 100755 index 0000000000..38799727d9 --- /dev/null +++ b/firebolt/install @@ -0,0 +1,6 @@ +#!/bin/bash +set -eu + +sudo apt-get update -y +sudo apt-get install -y docker.io jq +sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc diff --git a/firebolt/load b/firebolt/load new file mode 100755 index 0000000000..ba2864fe32 --- /dev/null +++ b/firebolt/load @@ -0,0 +1,20 @@ +#!/bin/bash +set -eu + +# Stage hits.parquet where the container can see it (./data is +# bind-mounted as /firebolt-core/clickbench). +mkdir -p data +if [ -f hits.parquet ]; then + mv -f hits.parquet data/hits.parquet +fi + +# create.sql CREATEs hits_external pointing at the parquet file, then +# INSERTs into the managed `hits` table — the ingested-to-Firebolt +# variant of the benchmark. +curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \ + --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;' +curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \ + --data-binary @create.sql + +rm -f data/hits.parquet +sync diff --git a/firebolt/query b/firebolt/query new file mode 100755 index 0000000000..910591e6b8 --- /dev/null +++ b/firebolt/query @@ -0,0 +1,28 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it against the firebolt-core +# container via /?database=clickbench. +# Stdout: query result (firebolt's JSON_Compact format). +# Stderr: query runtime in fractional seconds on the last line, +# pulled from the response's `.statistics.elapsed`. +# Exit non-zero on error. +set -e + +query=$(cat) + +# Result + sub-result caches off so timings are real; output_format +# matches what firebolt's run.sh uses for the public benchmark. +PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact' + +resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \ + --data-binary "$query") + +# Firebolt returns a JSON object whether the query succeeded or not. +# A failed query has an "errors" key; a successful one carries +# "data" + "statistics". +if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then + printf '%s\n' "$resp" >&2 + exit 1 +fi + +printf '%s\n' "$resp" +printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2 diff --git a/firebolt/run.sh b/firebolt/run.sh deleted file mode 100755 index 08bdfdbc18..0000000000 --- a/firebolt/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ "$1" != "" ] && [ "$1" != "scan-cache" ]; then - echo "Error: command line argument must be one of {'', 'scan-cache'}" - exit 1 -fi - -SCAN_CACHE="false" -if [ "$1" == "scan-cache" ]; then - SCAN_CACHE="true" -fi - -# Disable the result and subresult caches. Enable the scan-cache. -QUERY_PARAMS="enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=${SCAN_CACHE}&output_format=JSON_Compact" - -cat queries.sql | while read -r query; do - # Firebolt is a database with local on-disk storage: drop the page cache before the first run of each query. - sync - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - # Run the query three times. - # Extract the elapsed time from the response's statistics. - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n "[${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo -n ",${ELAPSED}" - ELAPSED=$(curl -sS "http://localhost:3473/?database=clickbench&${QUERY_PARAMS}" --data-binary "$query" | jq '.statistics.elapsed') - echo ",${ELAPSED}]," -done diff --git a/firebolt/start b/firebolt/start new file mode 100755 index 0000000000..652146394e --- /dev/null +++ b/firebolt/start @@ -0,0 +1,67 @@ +#!/bin/bash +set -eu + +# Idempotent: if firebolt-core already answers SELECT 1, do nothing. +if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'; then + exit 0 +fi + +mkdir -p data fb-volume +# firebolt-core runs as UID/GID 1111 inside the container and refuses +# to start if its data dir is not writeable by that uid (the engine +# self-checks and aborts with "directory ... is not readable or +# writeable by the Firebolt Core process"). Set the host-side +# ownership accordingly so the bind-mounted dir is usable. +sudo chown 1111:1111 fb-volume + +# If the container exists (stopped from a prior agent pre-snapshot +# cycle), just start it back — the data lives on the bind-mounted +# fb-volume below, so the previously-created `clickbench` database +# is still there. Otherwise create the container fresh. +if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then + sudo docker start firebolt-core >/dev/null +else + # `firebolt-core` is the public self-hosted image. Container needs + # memlock 8 GiB and seccomp unconfined per upstream's run docs. + # /firebolt-core/clickbench: parquet source (read at load time). + # /firebolt-core/volume: engine data directory (must persist + # across the agent's pre-snapshot + # stop+start cycle or the snapshot + # ships an empty DB). + sudo docker run -dit --name firebolt-core \ + --ulimit memlock=8589934592:8589934592 \ + --security-opt seccomp=unconfined \ + -p 127.0.0.1:3473:3473 \ + -v "$(pwd)/fb-volume:/firebolt-core/volume" \ + -v "$(pwd)/data:/firebolt-core/clickbench" \ + ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null +fi + +# Wait for the cluster to be "actually" ready. firebolt-core's HTTP +# port comes up immediately but returns +# {"errors":[{"description":"Cluster not yet healthy: ..."}]} +# at HTTP 200 until the engine threads have warmed; bench against a +# sentinel string instead of HTTP status to avoid that trap. +for _ in $(seq 1 600); do + if curl -sS --max-time 5 'http://localhost:3473/' \ + --data-binary "SELECT 'firebolt-ready';" 2>/dev/null \ + | grep -q 'firebolt-ready'; then + exit 0 + fi + sleep 1 +done +{ + echo "firebolt-core did not become healthy in 10 min" + echo "=== docker ps -a ===" + sudo docker ps -a 2>&1 + echo "=== docker inspect firebolt-core (state) ===" + sudo docker inspect firebolt-core --format '{{json .State}}' 2>&1 + echo "=== docker logs firebolt-core --tail 50 ===" + sudo docker logs firebolt-core --tail 50 2>&1 + echo "=== curl http://localhost:3473/ ===" + curl -sS --max-time 3 'http://localhost:3473/' --data-binary 'SELECT 1' 2>&1 + echo "=== ss listeners ===" + sudo ss -lntp 2>&1 | head -20 +} >&2 +exit 1 diff --git a/firebolt/stop b/firebolt/stop new file mode 100755 index 0000000000..ac1834f7d6 --- /dev/null +++ b/firebolt/stop @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Plain stop — leave the container in place so its bind-mounted +# fb-volume keeps the loaded database for the next ./start. The +# container is removed and the volume re-initialised only on +# explicit re-provision. +sudo docker container stop firebolt-core >/dev/null 2>&1 || true diff --git a/kinetica/load b/kinetica/load index 523f581545..1097ee3866 100755 --- a/kinetica/load +++ b/kinetica/load @@ -9,11 +9,23 @@ CLI="./kisql --host localhost --user admin" # decompressed TSV. wget --continue --progress=dot:giga \ 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' -# Symlink rather than copy: hits.tsv.gz is 16 GB and we only read it once. -sudo ln -sf "$PWD/hits.tsv.gz" ./kinetica-persist/hits.tsv.gz +# Move (rename) rather than symlink: the kinetica daemon runs inside a +# docker container with ./kinetica-persist bind-mounted, so a symlink +# pointing at $PWD/hits.tsv.gz dangles inside the container and `LOAD +# INTO ... FROM FILE PATHS 'hits.tsv.gz'` returns +# Not_Found: No such file(s) (File(s):hits.tsv.gz) +# The persist dir and $PWD live on the same overlay filesystem, so +# mv is a rename — cheap. +sudo mv hits.tsv.gz ./kinetica-persist/ $CLI --file create.sql -$CLI --sql "ALTER TIER ram WITH OPTIONS ('capacity' = '27000000000');" +# Playground VMs have 16 GiB RAM total. The upstream 27 GB cap was +# sized for a host-mode benchmark machine; in the VM the RAM tier +# alone exceeds physical memory, kinetica's rank-1 worker gets +# OOM-killed mid-LOAD, and the load fails with +# [GPUdb]executeSql: Internal_Error: Rank 1 non-responsive +# Cap the RAM tier at 9 GB and rely on the on-disk tier for the rest. +$CLI --sql "ALTER TIER ram WITH OPTIONS ('capacity' = '9000000000');" $CLI --sql "load into hits from file paths 'hits.tsv.gz' format delimited text (INCLUDES HEADER=false, DELIMITER = '\t') WITH OPTIONS (NUM_TASKS_PER_RANK=16, ON ERROR=SKIP);" diff --git a/pandas/.preserve-state b/pandas/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pandas/benchmark.sh b/pandas/benchmark.sh index 084c13353c..1369feb230 100755 --- a/pandas/benchmark.sh +++ b/pandas/benchmark.sh @@ -6,4 +6,9 @@ export BENCH_DURABLE=no # queries.sql holds those Python expressions, one per line, so the # default BENCH_QUERIES_FILE=queries.sql in lib/benchmark-common.sh # picks them up unchanged. +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/parseable/install b/parseable/install index 9fcb8ffa1d..2d61f7ead5 100755 --- a/parseable/install +++ b/parseable/install @@ -5,8 +5,15 @@ sudo apt-get update -y sudo apt-get install -y parallel pigz pv if [ ! -x ./parseable ]; then + # v2.5.12 has an Arrow type-inference bug under static-schema mode: + # bare JSON numbers get inferred as Float64, so every ingest of a + # row with an Int64-declared field returns 400 with "Fail to merge + # schema field 'X' because the from data_type = Float64 does not + # equal Int64". Net effect: 0 rows loaded, every query returns 0. + # v2.7.2 fixes the inference; verified locally end-to-end against + # the bundled static_schema.json and hits.json. wget --continue --progress=dot:giga \ - https://github.com/parseablehq/parseable/releases/download/v2.5.12/Parseable_OSS_x86_64-unknown-linux-gnu + https://github.com/parseablehq/parseable/releases/download/v2.7.2/Parseable_OSS_x86_64-unknown-linux-gnu mv Parseable_OSS_x86_64-unknown-linux-gnu parseable chmod +x parseable fi diff --git a/parseable/load b/parseable/load index 7249a5257a..45a7fb541d 100755 --- a/parseable/load +++ b/parseable/load @@ -3,16 +3,42 @@ set -eu NUM_CORES=$(nproc) -# hits.json is delivered by the shared lib/download-hits-json helper -# (symlink to the RO dataset on the playground, wget + pigz -# elsewhere). +# Prefer the pre-decompressed hits.json shipped on the playground's +# readonly dataset disk — it's a 217 GB symlink target, doesn't burn +# the VM's 200 GB sparse sysdisk on a redundant gunzip. Fall back to +# wget + gunzip for standalone use. +if [ -f /opt/clickbench/datasets_ro/hits.json ]; then + ln -sf /opt/clickbench/datasets_ro/hits.json hits.json +elif [ -f /opt/clickbench/datasets_ro/hits.json.gz ]; then + ln -sf /opt/clickbench/datasets_ro/hits.json.gz hits.json.gz + FILE_SIZE=$(stat -L -c %s hits.json.gz) + pv -s "$FILE_SIZE" hits.json.gz | pigz -d > hits.json +else + wget --continue --progress=dot:giga \ + 'https://datasets.clickhouse.com/hits_compatible/hits.json.gz' + FILE_SIZE=$(stat -L -c %s hits.json.gz) + pv -s "$FILE_SIZE" hits.json.gz | pigz -d > hits.json +fi -# Create the stream first — ingest below needs it to exist. -curl --silent --location --request PUT 'http://localhost:8000/api/v1/logstream/hits' \ +# Create the stream first — ingest below needs it to exist. Loud +# error reporting on purpose: the previous --silent + ignored exit +# code masked a 400 here for the entire load (every /ingest then +# returned 400 because the stream didn't exist, and the only +# evidence was 100k+ curl 400 lines in the provision log). +echo "==> creating logstream hits" +resp=$(curl --silent --show-error --location --request PUT \ + -w '\nHTTP_CODE=%{http_code}\n' \ + 'http://localhost:8000/api/v1/logstream/hits' \ -H 'X-P-Static-Schema-Flag: true' \ -H 'Content-Type: application/json' \ -u "admin:admin" \ - --data-binary @static_schema.json >/dev/null + --data-binary @static_schema.json) || true +printf '%s\n' "$resp" +code=$(printf '%s' "$resp" | awk -F= '/^HTTP_CODE=/ {print $2}' | tail -1) +if [ "${code:-}" != "200" ] && [ "${code:-}" != "201" ]; then + echo "parseable logstream create failed (HTTP $code)" >&2 + exit 1 +fi # Wrap each block of LINES_PER_CHUNK NDJSON lines in [ ... ] and POST # directly to /api/v1/ingest. Inlined into parallel's command string @@ -32,8 +58,8 @@ pv hits.json | parallel --pipe -N$LINES_PER_CHUNK --block 10M \ --data-binary @- >/dev/null ' -# Drop the symlink/file delivered by lib/download-hits-json. -rm -f hits.json +# Drop the symlink to the RO dataset — no chunk files to clean up. +rm -f hits.json hits.json.gz # Allow sync to complete. sleep 180 diff --git a/pinot/.preserve-state b/pinot/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pinot/benchmark.sh b/pinot/benchmark.sh index 1d14d0bb2c..2379871512 100755 --- a/pinot/benchmark.sh +++ b/pinot/benchmark.sh @@ -6,4 +6,9 @@ export BENCH_DURABLE=yes # inside one JVM and takes longer than the lib's 300 s default to be # query-ready on a cold instance. 900 s clears the observed cold start. export BENCH_CHECK_TIMEOUT=900 +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/polars-dataframe/.preserve-state b/polars-dataframe/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/polars-dataframe/benchmark.sh b/polars-dataframe/benchmark.sh index 90bdcf07e3..a5a40fb651 100755 --- a/polars-dataframe/benchmark.sh +++ b/polars-dataframe/benchmark.sh @@ -6,4 +6,9 @@ export BENCH_DURABLE=no # queries.sql holds those Python expressions, one per line, so the # default BENCH_QUERIES_FILE=queries.sql in lib/benchmark-common.sh # picks them up unchanged. +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/polars/benchmark.sh b/polars/benchmark.sh index 6bf667e4f0..4148581913 100755 --- a/polars/benchmark.sh +++ b/polars/benchmark.sh @@ -2,4 +2,9 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=no +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/polars/query b/polars/query index 9129884cf7..3aa96790e7 100755 --- a/polars/query +++ b/polars/query @@ -1,6 +1,7 @@ #!/bin/bash -# Reads a SQL query from stdin, dispatches to the running polars server. -# Stdout: server response JSON. +# Reads a polars expression from stdin, dispatches to the running +# polars server. +# Stdout: rendered result (the eval'd value as a string). # Stderr: query runtime in fractional seconds on the last line. # Exit non-zero on error. set -e @@ -19,5 +20,14 @@ if [ "$status" != "200" ]; then exit 1 fi -echo "$body" -echo "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin)["elapsed"])' >&2 +# Server returns {"elapsed": , "result": ""}. +# Print the rendered result to stdout, elapsed to stderr. +printf '%s\n' "$body" | python3 -c ' +import json, sys +o = json.load(sys.stdin) +r = o.get("result", "") +sys.stdout.write(r) +if r and not r.endswith("\n"): + sys.stdout.write("\n") +sys.stderr.write(str(o["elapsed"]) + "\n") +' diff --git a/polars/server.py b/polars/server.py index 73f5413795..8f01385202 100755 --- a/polars/server.py +++ b/polars/server.py @@ -64,9 +64,17 @@ async def query(request: Request): except SyntaxError as e: raise HTTPException(status_code=400, detail=f"syntax error: {e}") start = timeit.default_timer() - eval(compiled, {"hits": hits, "pl": pl, "date": date}) + value = eval(compiled, {"hits": hits, "pl": pl, "date": date}) elapsed = round(timeit.default_timer() - start, 3) - return {"elapsed": elapsed} + # Render the eval result so the playground UI shows something + # instead of just a timing line. polars DataFrames / Series / + # LazyFrames have a useful __str__; everything else (scalar, + # tuple, dict, ...) falls through repr. + if isinstance(value, (pl.DataFrame, pl.Series, pl.LazyFrame)): + result = str(value) + else: + result = repr(value) + return {"elapsed": elapsed, "result": result} @app.get("/data-size") diff --git a/presto-partitioned/start b/presto-partitioned/start index 92bbe10997..125a4ca19d 100755 --- a/presto-partitioned/start +++ b/presto-partitioned/start @@ -17,4 +17,5 @@ sudo docker run -d --name presto \ -v "$PWD/etc/jvm.config:/opt/presto-server/etc/jvm.config:ro" \ -v "$PWD/etc/config.properties:/opt/presto-server/etc/config.properties:ro" \ -v "$PWD/data:/clickbench" \ + -v "/opt/clickbench/datasets_ro:/opt/clickbench/datasets_ro:ro" \ prestodb/presto:${PRESTO_VERSION} diff --git a/quickwit/benchmark.sh b/quickwit/benchmark.sh index 3ac19604ca..cb593204ba 100755 --- a/quickwit/benchmark.sh +++ b/quickwit/benchmark.sh @@ -1,7 +1,14 @@ #!/bin/bash # Thin shim — actual flow is in lib/benchmark-common.sh. -# Quickwit takes Elasticsearch-format JSON queries. -export BENCH_DOWNLOAD_SCRIPT="download-hits-json" +# Quickwit takes Elasticsearch-format JSON queries; the load script fetches +# hits.json.gz directly so no shared download-hits-* script applies. +export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_QUERIES_FILE="queries.json" +# After firecracker snapshot+restore the cluster's +# internal connections (brpc/gossip) are stale; ./start's +# shallow health probe doesn't notice and short-circuits. +# Tell the playground agent to ./stop the cluster before +# ./start so the next bring-up is from a clean state. +export PLAYGROUND_RESTART_AFTER_RESTORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/siglens/README.md b/siglens/README.md index 074b89ff98..0c43bf91f0 100644 --- a/siglens/README.md +++ b/siglens/README.md @@ -3,4 +3,4 @@ This document outlines the process for running a benchmark on SigLens, a observa Note about queries: - SigLens does not support SQL but supports Splunk Query Language (SPL). The SQL queries used by the benchmark have been translated into the splunk query language. - To ensure the accuracy of the translated Splunk Query Language queries, each SQL query was executed against the same dataset in ClickHouse. The responses from SigLens and ClickHouse were compared, and all results were identical. -- Some of the original queries are not supported and not run by the benchmark. The corresponding results have been recorded as null in `queries.spl` and `results.csv` respectively. +- Some of the original queries are not supported and not run by the benchmark. The corresponding results have been recorded as null in `queries.sql` and `results.csv` respectively. diff --git a/siglens/benchmark.sh b/siglens/benchmark.sh index 6d6c4dead1..d57ece0bc8 100755 --- a/siglens/benchmark.sh +++ b/siglens/benchmark.sh @@ -1,7 +1,8 @@ #!/bin/bash # Thin shim — actual flow is in lib/benchmark-common.sh. -export BENCH_DOWNLOAD_SCRIPT="download-hits-json" +# siglens ingests its own gzipped NDJSON; ./load fetches it directly. +export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes # queries are SPL/Splunk QL, not SQL. -export BENCH_QUERIES_FILE="queries.spl" +export BENCH_QUERIES_FILE="queries.sql" exec ../lib/benchmark-common.sh diff --git a/siglens/queries.spl b/siglens/queries.sql similarity index 100% rename from siglens/queries.spl rename to siglens/queries.sql diff --git a/siglens/query b/siglens/query index 1ca8b5a918..a1fe18284a 100755 --- a/siglens/query +++ b/siglens/query @@ -7,7 +7,7 @@ set -e querytxt=$(cat) -# A "null" query in queries.spl means "not supported"; emit null timing. +# A "null" query in queries.sql means "not supported"; emit null timing. if [ "$querytxt" = "null" ]; then echo "{}" echo "null" >&2 diff --git a/tidb/.preserve-state b/tidb/.preserve-state deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tidb/benchmark.sh b/tidb/benchmark.sh index 107b9dbb65..73f1c4ad83 100755 --- a/tidb/benchmark.sh +++ b/tidb/benchmark.sh @@ -3,4 +3,9 @@ # TiDB Lightning loads from ..csv files; we use the CSV download. export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes +# Skip the pre-snapshot ./stop+./start cycle: the loaded +# state lives only in the daemon's process memory (in-process +# DataFrame, JVM heap caches) and stopping wipes it. The +# playground agent reads this and snapshots the running daemon. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/trino-datalake-partitioned/benchmark.sh b/trino-datalake-partitioned/benchmark.sh index 63f96c9b51..64f93a400d 100755 --- a/trino-datalake-partitioned/benchmark.sh +++ b/trino-datalake-partitioned/benchmark.sh @@ -3,4 +3,6 @@ # Datalake variant: Parquet is read directly from public S3, no download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes +# Trino bootstrap on a cold sysdisk pushes past the 300s default. +export BENCH_CHECK_TIMEOUT=3600 exec ../lib/benchmark-common.sh diff --git a/trino-datalake/benchmark.sh b/trino-datalake/benchmark.sh index 63f96c9b51..64f93a400d 100755 --- a/trino-datalake/benchmark.sh +++ b/trino-datalake/benchmark.sh @@ -3,4 +3,6 @@ # Datalake variant: Parquet is read directly from public S3, no download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes +# Trino bootstrap on a cold sysdisk pushes past the 300s default. +export BENCH_CHECK_TIMEOUT=3600 exec ../lib/benchmark-common.sh diff --git a/trino-datalake/start b/trino-datalake/start index 5e66109ab9..ed6cd7eab3 100755 --- a/trino-datalake/start +++ b/trino-datalake/start @@ -16,3 +16,19 @@ sudo docker run -d --name trino \ -v "$PWD/data/meta:/data/meta" \ -v "$PWD/shim/S3AnonymousProvider.jar:/usr/lib/trino/plugin/hive/hdfs/S3AnonymousProvider.jar:ro" \ trinodb/trino:455 + +# One-shot diagnostic: trino is slow to come up and the agent's +# check loop only watches /v1/info. If the container died or trino +# is stuck in startup, we want a snapshot of container state in the +# provision log so the failure is debuggable. +sleep 30 +echo "=== trino-datalake: docker ps -a ===" +sudo docker ps -a 2>&1 | head -10 +echo "=== trino-datalake: trino state ===" +sudo docker inspect trino --format \ + 'Status={{.State.Status}} Running={{.State.Running}} ExitCode={{.State.ExitCode}} OOMKilled={{.State.OOMKilled}}' 2>&1 +echo "=== trino-datalake: docker logs --tail 80 ===" +sudo docker logs trino --tail 80 2>&1 +echo "=== trino-datalake: /v1/info ===" +curl -sS --max-time 3 http://localhost:8080/v1/info 2>&1 | head -c 500 +echo diff --git a/trino-partitioned/start b/trino-partitioned/start index da87d704b4..b07580d780 100755 --- a/trino-partitioned/start +++ b/trino-partitioned/start @@ -13,4 +13,5 @@ sudo docker run -d --name trino \ -p 8080:8080 \ -v "$PWD/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties:ro" \ -v "$PWD/data:/clickbench" \ + -v "/opt/clickbench/datasets_ro:/opt/clickbench/datasets_ro:ro" \ trinodb/trino:latest diff --git a/umbra/start b/umbra/start index eafb1ebcb4..097802346f 100755 --- a/umbra/start +++ b/umbra/start @@ -32,11 +32,44 @@ sudo docker run -d --name umbradb \ -v "$(pwd)/data:/data" \ -p 5432:5432 \ --ulimit nofile=1048576:1048576 \ - --ulimit memlock=8388608:8388608 \ + --ulimit memlock=-1:-1 \ umbradb/umbra:latest >/dev/null # Container needs a moment before psql can connect. for _ in $(seq 1 60); do - PGPASSWORD=postgres psql -p 5432 -h 127.0.0.1 -U postgres -c 'SELECT 1' >/dev/null 2>&1 && exit 0 + if PGPASSWORD=postgres psql -p 5432 -h 127.0.0.1 -U postgres \ + -c 'SELECT 1' >/dev/null 2>&1; then + # Diagnostic dump so a future OOM during load lands with the + # memory/swap state of the VM in the provision log. Previously + # silent — every "unable to allocate memory" failure looked + # the same and we couldn't tell whether the agent's mkswap+ + # swapon ran, whether the container saw the swap, or whether + # the sysctl tweaks above stuck. + echo "=== umbra: VM memory state ===" + free -h || true + echo "=== umbra: swap state ===" + swapon --show=NAME,SIZE,USED,PRIO --bytes || true + echo "=== umbra: sysctl ===" + for k in vm.overcommit_memory vm.swappiness vm.max_map_count \ + vm.overcommit_ratio; do + echo " $k = $(sysctl -n $k 2>/dev/null)" + done + echo "=== umbra: container memory cgroup ===" + sudo docker inspect umbradb --format \ + 'memory={{.HostConfig.Memory}} memory-swap={{.HostConfig.MemorySwap}}' || true + echo "=== umbra: container memlock ulimit ===" + sudo docker exec umbradb sh -c 'ulimit -l' 2>&1 || true + cgpath=$(sudo docker inspect umbradb --format '{{.State.Pid}}' 2>/dev/null | \ + xargs -I{} cat /proc/{}/cgroup 2>/dev/null | awk -F: '{print $NF}') + if [ -n "$cgpath" ]; then + for f in memory.max memory.swap.max memory.swap.current; do + p="/sys/fs/cgroup${cgpath}/$f" + [ -r "$p" ] && echo " $f = $(cat "$p")" + done + fi + echo "=== umbra: container procs ===" + sudo docker top umbradb -eo pid,vsz,rss,comm 2>&1 | head -10 + exit 0 + fi sleep 1 done