diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index d7534297d5afb..99e5f648a894d 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -19,7 +19,6 @@ # /ci-run-component-features : runs Component Features - Linux # /ci-run-unit-mac : runs Unit - Mac # /ci-run-unit-windows : runs Unit - Windows -# /ci-run-environment : runs Environment Suite # /ci-run-k8s : runs K8s E2E Suite # /ci-run-packaging : builds and verifies packages (without publishing to Docker/S3/GitHub) # @@ -79,7 +78,6 @@ jobs: || startsWith(github.event.review.body, '/ci-run-cross') || startsWith(github.event.review.body, '/ci-run-unit-mac') || startsWith(github.event.review.body, '/ci-run-unit-windows') - || startsWith(github.event.review.body, '/ci-run-environment') || startsWith(github.event.review.body, '/ci-run-k8s') || startsWith(github.event.review.body, '/ci-run-packaging') || startsWith(github.event.review.body, '/ci-run-regression') @@ -194,14 +192,6 @@ jobs: ref: ${{ github.event.review.commit_id }} secrets: inherit - environment: - needs: validate - if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-environment') - uses: ./.github/workflows/environment.yml - with: - ref: ${{ github.event.review.commit_id }} - secrets: inherit - k8s: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-k8s') diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml deleted file mode 100644 index a989886548df6..0000000000000 --- a/.github/workflows/environment.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Environment Suite - -on: - workflow_call: - inputs: - ref: - description: "Git ref to checkout" - required: false - type: string - - workflow_dispatch: - inputs: - push: - description: "Push image when manually triggered" - type: boolean - default: false - ref: - description: "Git ref to checkout" - required: false - type: string - - schedule: - - cron: "0 6 * * 1" # Every Monday at 06:00 UTC - -env: - VERBOSE: true - CI: true - SHOULD_PUBLISH: ${{ (github.event_name != 'workflow_dispatch' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.push == 'true') }} - -permissions: - contents: read - -jobs: - publish-new-environment: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.ref }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - name: Login to DockerHub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - if: env.SHOULD_PUBLISH == 'true' - with: - username: ${{ secrets.CI_DOCKER_USERNAME }} - password: ${{ secrets.CI_DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 - with: - images: timberio/vector-dev - flavor: | - latest=true - tags: type=sha, format=long - labels: | - org.opencontainers.image.description=Image for Vector's Docker development environment - org.opencontainers.image.source=https://github.com/vectordotdev/vector/tree/master/scripts/environment/Dockerfile - org.opencontainers.image.title=Vector development environment - org.opencontainers.image.url=https://github.com/vectordotdev/vector - - - name: Free disk space - shell: bash - run: sudo -E bash scripts/ci-free-disk-space.sh - - - name: Build image (no push) - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - file: ./scripts/environment/Dockerfile - load: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Test built image - run: | - IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - echo "Running build test inside container: ${IMAGE_TAG}" - docker run --rm \ - -v "${GITHUB_WORKSPACE}:/src" \ - -w /src \ - "${IMAGE_TAG}" \ - cargo run --bin vector -- --version - - - name: Push image - if: env.SHOULD_PUBLISH == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - file: ./scripts/environment/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/Makefile b/Makefile index 905798634b2cd..67aac49ffb60a 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) mkfile_dir := $(dir $(mkfile_path)) +# Make project-local npm tools (installed by scripts/environment/prepare.sh on +# laptops) discoverable to recipes without requiring contributors to edit PATH. +# CI installs the same tools globally and is unaffected by the prefix. +export PATH := $(mkfile_dir)scripts/environment/npm-tools/node_modules/.bin:$(PATH) + # Begin OS detection ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... export OPERATING_SYSTEM := Windows @@ -54,33 +59,9 @@ endif # If we're using podman create pods else if we're using docker create networks. export CURRENT_DIR = $(shell pwd) -# Override this to automatically enter a container containing the correct, full, official build environment for Vector, ready for development -export ENVIRONMENT ?= false -# The upstream container we publish artifacts to on a successful master build. -export ENVIRONMENT_UPSTREAM ?= docker.io/timberio/vector-dev:latest -# Override to disable building the container, having it pull from the GitHub packages repo instead -# TODO: Disable this by default. Blocked by `docker pull` from GitHub Packages requiring authenticated login -export ENVIRONMENT_AUTOBUILD ?= true -# Override to disable force pulling the image, leaving the container tool to pull it only when necessary instead -export ENVIRONMENT_AUTOPULL ?= true -# Override this when appropriate to disable a TTY being available in commands with `ENVIRONMENT=true` -export ENVIRONMENT_TTY ?= true -# Override to specify which network the environment will be connected to (leave empty to use the container tool default) -export ENVIRONMENT_NETWORK ?= host -# Override to specify environment port(s) to publish to the host (leave empty to not configure any port publishing) -# Multiple port publishing can be provided using spaces, for example: 8686:8686 8080:8080/udp -export ENVIRONMENT_PUBLISH ?= - -# If ENVIRONMENT is true, always use cargo vdev since it may be running inside the container -ifeq ($(origin VDEV), environment) -ifeq ($(ENVIRONMENT), true) -VDEV := cargo vdev -else -# VDEV is already set from environment, keep it -endif -else -VDEV := cargo vdev -endif +# Preserve any caller-supplied VDEV (e.g. CI exports the pinned prebuilt binary +# via .github/actions/setup; falling back to `cargo vdev` recompiles vdev). +VDEV ?= cargo vdev # Set dummy AWS credentials if not present - used for AWS and ES integration tests export AWS_ACCESS_KEY_ID ?= "dummy" @@ -109,117 +90,23 @@ help: @printf -- " V E C T O R\n" @printf -- "\n" @printf -- "---------------------------------------------------------------------------------------\n" - @printf -- "Want to use ${FORMATTING_BEGIN_YELLOW}\`docker\`${FORMATTING_END} or ${FORMATTING_BEGIN_YELLOW}\`podman\`${FORMATTING_END}? See ${FORMATTING_BEGIN_YELLOW}\`ENVIRONMENT=true\`${FORMATTING_END} commands. (Default ${FORMATTING_BEGIN_YELLOW}\`CONTAINER_TOOL=docker\`${FORMATTING_END})\n" + @printf -- "Default ${FORMATTING_BEGIN_YELLOW}\`CONTAINER_TOOL=docker\`${FORMATTING_END} (auto-detects ${FORMATTING_BEGIN_YELLOW}\`docker\`${FORMATTING_END} or ${FORMATTING_BEGIN_YELLOW}\`podman\`${FORMATTING_END}).\n" @printf -- "\n" @awk 'BEGIN {FS = ":.*##"; printf "Usage: make ${FORMATTING_BEGIN_BLUE}${FORMATTING_END}\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " ${FORMATTING_BEGIN_BLUE}%-46s${FORMATTING_END} %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) ##@ Environment -# These are some predefined macros, please use them! -ifeq ($(ENVIRONMENT), true) -define MAYBE_ENVIRONMENT_EXEC -${ENVIRONMENT_EXEC} -endef -else -define MAYBE_ENVIRONMENT_EXEC - -endef -endif - -ifeq ($(ENVIRONMENT), true) -define MAYBE_ENVIRONMENT_COPY_ARTIFACTS -${ENVIRONMENT_COPY_ARTIFACTS} -endef -else -define MAYBE_ENVIRONMENT_COPY_ARTIFACTS - -endef -endif - -# docker container id file needs to live in the host machine and is later mounted into the container -CIDFILE := $(shell mktemp -u /tmp/vector-environment-docker-cid.XXXXXX) - -# We use a volume here as non-Linux hosts are extremely slow to share disks, and Linux hosts tend to get permissions clobbered. -define ENVIRONMENT_EXEC - ${ENVIRONMENT_PREPARE} - @echo "Entering environment..." - @mkdir -p target - $(CONTAINER_TOOL) run \ - --name vector-environment \ - --rm \ - $(if $(findstring true,$(ENVIRONMENT_TTY)),--tty,) \ - --init \ - --interactive \ - --env INSIDE_ENVIRONMENT=true \ - $(if $(ENVIRONMENT_NETWORK),--network $(ENVIRONMENT_NETWORK),) \ - --mount type=bind,source=${CURRENT_DIR},target=/git/vectordotdev/vector \ - $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \ - $(if $(findstring docker,$(CONTAINER_TOOL)),--cidfile $(CIDFILE),) \ - $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=$(CIDFILE)$(COMMA)target=/.docker-container-id,) \ - --mount type=volume,source=vector-target,target=/git/vectordotdev/vector/target \ - --mount type=volume,source=vector-cargo-cache,target=/root/.cargo \ - --mount type=volume,source=vector-rustup-cache,target=/root/.rustup \ - $(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \ - $(ENVIRONMENT_UPSTREAM); rm -f $(CIDFILE) -endef - - -ifneq ($(CONTAINER_TOOL), unknown) -ifeq ($(ENVIRONMENT_AUTOBUILD), true) -define ENVIRONMENT_PREPARE - @echo "Building the environment. (ENVIRONMENT_AUTOBUILD=true) This may take a few minutes..." - $(CONTAINER_TOOL) build \ - $(if $(findstring true,$(VERBOSE)),,--quiet) \ - --tag $(ENVIRONMENT_UPSTREAM) \ - --file scripts/environment/Dockerfile \ - . -endef -else ifeq ($(ENVIRONMENT_AUTOPULL), true) -define ENVIRONMENT_PREPARE - @echo "Pulling the environment image. (ENVIRONMENT_AUTOPULL=true)" - $(CONTAINER_TOOL) pull $(ENVIRONMENT_UPSTREAM) -endef -endif -else -define ENVIRONMENT_PREPARE -$(error "Please install a container tool such as Docker or Podman") -endef -endif - .PHONY: check-container-tool check-container-tool: ## Checks what container tool is installed @echo -n "Checking if $(CONTAINER_TOOL) is available..." && \ $(CONTAINER_TOOL) version 1>/dev/null && echo "yes" -.PHONY: environment -environment: export ENVIRONMENT_TTY = true ## Enter a full Vector dev shell in $CONTAINER_TOOL, binding this folder to the container. -environment: - ${ENVIRONMENT_EXEC} - -.PHONY: environment-prepare -environment-prepare: ## Prepare the Vector dev shell using $CONTAINER_TOOL. - ${ENVIRONMENT_PREPARE} - -.PHONY: environment-clean -environment-clean: ## Clean the Vector dev shell using $CONTAINER_TOOL. - @$(CONTAINER_TOOL) volume rm -f vector-target vector-cargo-cache vector-rustup-cache - @$(CONTAINER_TOOL) rmi $(ENVIRONMENT_UPSTREAM) || true - -.PHONY: environment-push -environment-push: environment-prepare ## Publish a new version of the container image. - $(CONTAINER_TOOL) push $(ENVIRONMENT_UPSTREAM) - ##@ Building .PHONY: build build: check-build-tools build: export CFLAGS += -g0 -O3 -build: ## Build the project in release mode (Supports `ENVIRONMENT=true`) - ${MAYBE_ENVIRONMENT_EXEC} cargo build --release --no-default-features --features ${FEATURES} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} - -.PHONY: build-dev -build-dev: ## Build the project in development mode (Supports `ENVIRONMENT=true`) - ${MAYBE_ENVIRONMENT_EXEC} cargo build --no-default-features --features ${FEATURES} +build: ## Build the project in release mode + cargo build --release --no-default-features --features ${FEATURES} .PHONY: build-x86_64-unknown-linux-gnu build-x86_64-unknown-linux-gnu: target/x86_64-unknown-linux-gnu/release/vector ## Build a release binary for the x86_64-unknown-linux-gnu triple. @@ -253,17 +140,11 @@ build-arm-unknown-linux-gnueabi: target/arm-unknown-linux-gnueabi/release/vector build-arm-unknown-linux-musleabi: target/arm-unknown-linux-musleabi/release/vector ## Build a release binary for the arm-unknown-linux-musleabi triple. @echo "Output to ${<}" -.PHONY: build-graphql-schema -build-graphql-schema: ## Generate the `schema.json` for Vector's GraphQL API - ${MAYBE_ENVIRONMENT_EXEC} cargo run --bin graphql-schema --no-default-features --features=default-no-api-client - .PHONY: check-build-tools check-build-tools: -ifneq ($(ENVIRONMENT), true) ifeq ($(shell command -v cargo >/dev/null || echo not-found), not-found) $(error "Please install Rust: https://www.rust-lang.org/tools/install") endif -endif ##@ Cross Compiling .PHONY: cross-enable @@ -366,7 +247,7 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS ./vector-${TRIPLE} rm -rf target/scratch/ -##@ Testing (Supports `ENVIRONMENT=true`) +##@ Testing # nextest doesn't support running doc tests yet so this is split out as # `test-docs` @@ -381,11 +262,11 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS # https://github.com/rust-lang/cargo/issues/6454 .PHONY: test test: ## Run the unit test suite - ${MAYBE_ENVIRONMENT_EXEC} ${TEST_RUNNER} --workspace --no-fail-fast --no-default-features --features "${FEATURES}" ${SCOPE} + ${TEST_RUNNER} --workspace --no-fail-fast --no-default-features --features "${FEATURES}" ${SCOPE} .PHONY: test-docs test-docs: ## Run the docs test suite - ${MAYBE_ENVIRONMENT_EXEC} cargo test --doc --workspace --no-fail-fast --no-default-features --features "${FEATURES}" ${SCOPE} + cargo test --doc --workspace --no-fail-fast --no-default-features --features "${FEATURES}" ${SCOPE} .PHONY: test-all test-all: test test-docs test-behavior test-integration test-component-validation ## Runs all tests: unit, docs, behavioral, integration, and component validation. @@ -400,12 +281,12 @@ test-aarch64-unknown-linux-gnu: cross-test-aarch64-unknown-linux-gnu ## Runs uni .PHONY: test-behavior-config test-behavior-config: ## Runs configuration related behavioral tests - ${MAYBE_ENVIRONMENT_EXEC} cargo build --no-default-features --features secret-backend-example --bin secret-backend-example - ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms -- test tests/behavior/config/* + cargo build --no-default-features --features secret-backend-example --bin secret-backend-example + cargo run --no-default-features --features transforms -- test tests/behavior/config/* .PHONY: test-behavior-% test-behavior-%: ## Runs behavioral test for a given category - ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network,vrl-functions-crypto -- test tests/behavior/$*/* + cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network,vrl-functions-crypto -- test tests/behavior/$*/* .PHONY: test-behavior test-behavior: ## Runs all behavioral tests @@ -424,7 +305,7 @@ test-integration: test-integration-datadog-traces test-integration-shutdown .PHONY: test-integration-windows-event-log test-integration-windows-event-log: ## Runs Windows Event Log integration tests (Windows only) ifeq ($(OS),Windows_NT) - ${MAYBE_ENVIRONMENT_EXEC} cargo test -p vector --no-default-features --features sources-windows_event_log-integration-tests windows_event_log::integration_tests + cargo test -p vector --no-default-features --features sources-windows_event_log-integration-tests windows_event_log::integration_tests else @echo "Skipping windows-event-log integration tests (Windows only)" endif @@ -439,78 +320,69 @@ ifeq ($(AUTODESPAWN), true) endif .PHONY: test-e2e-kubernetes -test-e2e-kubernetes: ## Runs Kubernetes E2E tests (Sorry, no `ENVIRONMENT=true` support) +test-e2e-kubernetes: ## Runs Kubernetes E2E tests RUST_VERSION=${RUST_VERSION} scripts/test-e2e-kubernetes.sh .PHONY: test-cli test-cli: ## Runs cli tests - ${MAYBE_ENVIRONMENT_EXEC} ${TEST_RUNNER} --no-fail-fast --no-default-features --features cli-tests --test integration --test-threads 4 + ${TEST_RUNNER} --no-fail-fast --no-default-features --features cli-tests --test integration --test-threads 4 .PHONY: test-vector-api test-vector-api: ## Runs vector API tests (top and tap) - ${MAYBE_ENVIRONMENT_EXEC} ${TEST_RUNNER} --no-fail-fast --no-default-features --features vector-api-tests --test vector_api + ${TEST_RUNNER} --no-fail-fast --no-default-features --features vector-api-tests --test vector_api .PHONY: test-component-validation test-component-validation: ## Runs component validation tests - ${MAYBE_ENVIRONMENT_EXEC} ${TEST_RUNNER} --no-fail-fast --no-default-features --features component-validation-tests --status-level pass --test-threads 4 --lib components::validation::tests + ${TEST_RUNNER} --no-fail-fast --no-default-features --features component-validation-tests --status-level pass --test-threads 4 --lib components::validation::tests .PHONY: coverage-report coverage-report: ## Generate lcov report after running tests with COVERAGE=true (outputs lcov.info) cargo llvm-cov report --lcov --output-path lcov.info -##@ Benching (Supports `ENVIRONMENT=true`) +##@ Benching .PHONY: bench bench: ## Run benchmarks in /benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "benches" ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "benches" ${CARGO_BENCH_FLAGS} .PHONY: bench-dnstap bench-dnstap: ## Run dnstap benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "dnstap-benches" --bench dnstap ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "dnstap-benches" --bench dnstap ${CARGO_BENCH_FLAGS} .PHONY: bench-dnsmsg-parser bench-dnsmsg-parser: ## Run dnsmsg-parser benches - ${MAYBE_ENVIRONMENT_EXEC} CRITERION_HOME="$(CRITERION_HOME)" cargo bench --manifest-path lib/dnsmsg-parser/Cargo.toml ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + CRITERION_HOME="$(CRITERION_HOME)" cargo bench --manifest-path lib/dnsmsg-parser/Cargo.toml ${CARGO_BENCH_FLAGS} .PHONY: bench-remap-functions bench-remap-functions: ## Run remap-functions benches - ${MAYBE_ENVIRONMENT_EXEC} CRITERION_HOME="$(CRITERION_HOME)" cargo bench --manifest-path lib/vrl/stdlib/Cargo.toml ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + CRITERION_HOME="$(CRITERION_HOME)" cargo bench --manifest-path lib/vrl/stdlib/Cargo.toml ${CARGO_BENCH_FLAGS} .PHONY: bench-remap bench-remap: ## Run remap benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "remap-benches" --bench remap ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "remap-benches" --bench remap ${CARGO_BENCH_FLAGS} .PHONY: bench-transform bench-transform: ## Run transform benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "transform-benches" --bench transform ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "transform-benches" --bench transform ${CARGO_BENCH_FLAGS} .PHONY: bench-languages bench-languages: ### Run language comparison benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "language-benches" --bench languages ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "language-benches" --bench languages ${CARGO_BENCH_FLAGS} .PHONY: bench-metrics bench-metrics: ## Run metrics benches - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "metrics-benches" ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "metrics-benches" ${CARGO_BENCH_FLAGS} .PHONY: bench-all bench-all: ### Run all benches bench-all: bench-remap-functions - ${MAYBE_ENVIRONMENT_EXEC} cargo bench --no-default-features --features "benches remap-benches metrics-benches language-benches ${DNSTAP_BENCHES}" ${CARGO_BENCH_FLAGS} - ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} + cargo bench --no-default-features --features "benches remap-benches metrics-benches language-benches ${DNSTAP_BENCHES}" ${CARGO_BENCH_FLAGS} ##@ Checking .PHONY: check check: ## Run prerequisite code checks - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust + $(VDEV) check rust .PHONY: check-all check-all: ## Check everything @@ -520,38 +392,38 @@ check-all: check-scripts check-deny check-generated-docs check-licenses .PHONY: check-component-features check-component-features: ## Check that all component features are setup properly - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check component-features + $(VDEV) check component-features .PHONY: check-clippy check-clippy: ## Check code with Clippy - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust + $(VDEV) check rust .PHONY: check-docs check-docs: generate-vrl-docs ## Check that all /docs file are valid - vrl docs due to remap.functions.* references - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check docs + $(VDEV) check docs .PHONY: check-fmt check-fmt: ## Check that all files are formatted properly - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check fmt + $(VDEV) check fmt .PHONY: check-licenses check-licenses: ## Check that the 3rd-party license file is up to date - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check licenses + $(VDEV) check licenses .PHONY: check-markdown check-markdown: ## Check that markdown is styled properly - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check markdown + $(VDEV) check markdown .PHONY: fix-markdown fix-markdown: ## Auto-fix markdown style issues - ${MAYBE_ENVIRONMENT_EXEC} markdownlint-cli2 --fix $(shell git ls-files '*.md') + markdownlint-cli2 --fix $(shell git ls-files '*.md') .PHONY: check-prettier check-prettier: ## Check that JS/TS/YAML/JSON files are formatted with prettier @for ext in yml yaml js ts tsx json; do \ files=$$(git ls-files "*.$$ext"); \ if [ -n "$$files" ]; then \ - ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --check $$files || exit 1; \ + prettier --ignore-path .prettierignore --check $$files || exit 1; \ fi; \ done @@ -560,38 +432,38 @@ fix-prettier: ## Auto-fix JS/TS/YAML/JSON formatting with prettier @for ext in yml yaml js ts tsx json; do \ files=$$(git ls-files "*.$$ext"); \ if [ -n "$$files" ]; then \ - ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --write $$files || exit 1; \ + prettier --ignore-path .prettierignore --write $$files || exit 1; \ fi; \ done .PHONY: check-examples check-examples: ## Check that the config/examples files are valid - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check examples + $(VDEV) check examples .PHONY: check-scripts check-scripts: ## Check that scripts do not have common mistakes - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check scripts + $(VDEV) check scripts .PHONY: check-deny check-deny: ## Check advisories licenses and sources for crate dependencies - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny + $(VDEV) check deny .PHONY: check-deny-licenses check-deny-licenses: ## Check licenses for crate dependencies - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny --licenses-only + $(VDEV) check deny --licenses-only .PHONY: check-events check-events: ## Check that events satisfy patterns set in https://github.com/vectordotdev/vector/blob/master/rfcs/2020-03-17-2064-event-driven-observability.md - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check events + $(VDEV) check events .PHONY: check-generated-docs check-generated-docs: generate-docs ## Checks that the machine-generated component Cue docs are up-to-date. - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check generated-docs + $(VDEV) check generated-docs ##@ Rustdoc build-rustdoc: ## Build Vector's Rustdocs # This command is mostly intended for use by the build process in vectordotdev/vector-rustdoc - ${MAYBE_ENVIRONMENT_EXEC} cargo doc --no-deps --workspace + cargo doc --no-deps --workspace ##@ Packaging (forwarded to Makefile.packaging) @@ -600,7 +472,7 @@ build-rustdoc: ## Build Vector's Rustdocs .PHONY: package package: build ## Build the Vector archive - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) package archive + $(VDEV) package archive package-%: $(MAKE) -f Makefile.packaging $@ @@ -644,7 +516,7 @@ compile-vrl-wasm: ## Compile VRL crates to WASM target ##@ Utility .PHONY: clean -clean: environment-clean ## Clean everything +clean: ## Clean everything cargo clean .PHONY: generate-kubernetes-manifests @@ -653,9 +525,9 @@ generate-kubernetes-manifests: ## Generate Kubernetes manifests from latest Helm .PHONY: generate-component-docs generate-component-docs: ## Generate per-component Cue docs from the configuration schema. - ${MAYBE_ENVIRONMENT_EXEC} cargo build $(if $(findstring true,$(CI)),--quiet,) + cargo build $(if $(findstring true,$(CI)),--quiet,) target/debug/vector generate-schema > /tmp/vector-config-schema.json 2>/dev/null - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build component-docs /tmp/vector-config-schema.json \ + $(VDEV) build component-docs /tmp/vector-config-schema.json \ $(if $(findstring true,$(CI)),>/dev/null,) ./scripts/cue.sh fmt @@ -668,7 +540,7 @@ endif .PHONY: generate-vector-vrl-docs generate-vector-vrl-docs: ## Generate VRL function documentation from Rust source. - ${MAYBE_ENVIRONMENT_EXEC} $(VRL_DOC_BUILDER_CMD) --output docs/generated/ \ + $(VRL_DOC_BUILDER_CMD) --output docs/generated/ \ $(if $(findstring true,$(CI)),>/dev/null,) .PHONY: generate-vrl-docs @@ -701,12 +573,12 @@ ci-generate-publish-metadata: ## Generates the necessary metadata required for b .PHONY: clippy-fix clippy-fix: - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust --fix + $(VDEV) check rust --fix .PHONY: fmt fmt: - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) fmt + $(VDEV) fmt .PHONY: build-licenses build-licenses: - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build licenses + $(VDEV) build licenses diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 9d3d50f7d6243..b22e78a1d8e11 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -1,8 +1,6 @@ # Developing - [Setup](#setup) - - [Using a Docker or Podman environment](#using-a-docker-or-podman-environment) - - [Bring your own toolbox](#bring-your-own-toolbox) - [The basics](#the-basics) - [Directory structure](#directory-structure) - [Makefile](#makefile) @@ -41,82 +39,7 @@ ## Setup -We're super excited to have you interested in working on Vector! Before you start you should pick how you want to develop. - -For small or first-time contributions, we recommend the Docker method. Prefer to do it yourself? That's fine too! - -### Using a Docker or Podman environment - -> **Targets:** You can use this method to produce AARCH64, Arm6/7, as well as x86/64 Linux builds. - -Since not everyone has a full working native environment, we took our environment and stuffed it into a Docker (or Podman) container! - -This is ideal for users who want it to "Just work" and just want to start contributing. It's also what we use for our CI, so you know if it breaks we can't do anything else until we fix it. 😉 - -**Before you go further, install Docker or Podman through your official package manager, or from the [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/) sites.** - -```bash -# Optional: Only if you use `podman` -export CONTAINER_TOOL="podman" -``` - -If your Linux environment runs SELinux in Enforcing mode, you will need to relabel the vector source code checkout with `container_home_t` context. Otherwise, the container environment cannot read/write the code: - -```bash -cd your/checkout/of/vector/ -sudo semanage fcontext -a "${PWD}(/.*)?" -t container_file_t -sudo restorecon . -R -``` - -By default, `make environment` style tasks will do a `docker pull` from GitHub's container repository, you can **optionally** build your own environment while you make your morning coffee ☕: - -```bash -# Optional: Only if you want to go make a coffee -make environment-prepare -``` - -Now that you have your coffee, you can enter the shell! - -```bash -# Enter a shell with optimized mounts for interactive processes. -# Inside here, you can use Vector like you have full toolchain (See below!) -make environment -# Try out a specific container tool. (Docker/Podman) -make environment CONTAINER_TOOL="podman" -# Add extra cli opts -make environment CLI_OPTS="--publish 3000:2000" -``` - -Now you can use the jobs detailed in **"Bring your own toolbox"** below. - -Want to run from outside of the environment? _Clever. Good thinking._ You can run any of the following: - -```bash -# Validate your code can compile -make check ENVIRONMENT=true -# Validate your code actually does compile (in dev mode) -make build-dev ENVIRONMENT=true -# Validate your test pass -make test SCOPE="sources::example" ENVIRONMENT=true -# Validate tests (that do not require other services) pass -make test ENVIRONMENT=true -# Validate your tests pass (starting required services in Docker) -make test-integration SCOPE="sources::example" ENVIRONMENT=true -# Validate your tests pass against a live service. -make test-integration SCOPE="sources::example" AUTOSPAWN=false ENVIRONMENT=true -# Validate all tests pass (starting required services in Docker) -make test-integration ENVIRONMENT=true -# Run your benchmarks -make bench SCOPE="transforms::example" ENVIRONMENT=true -# Format your code before pushing! -make fmt ENVIRONMENT=true -``` - -We use explicit environment opt-in as many contributors choose to keep their Rust toolchain local. - -### Bring your own toolbox - -> **Targets:** This option is required for MSVC/Mac/FreeBSD toolchains. It can be used to build for any environment or OS. +We're super excited to have you interested in working on Vector! To build Vector on your own host will require a fairly complete development environment! @@ -142,7 +65,13 @@ Loosely, you'll need the following: - **To run `make check-licenses` or `make build-licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). - **To run `make generate-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). -If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. +**Tooling shortcut:** Once the system-level dependencies above are in place, you can install the Rust and npm tooling (`cargo-nextest`, `cargo-deny`, `dd-rust-license-tool`, `vdev`, `markdownlint-cli2`, `prettier`, and others) in one shot: + +```bash +bash scripts/environment/prepare.sh +``` + +This is the same script CI uses to provision its toolchain, so what you get locally matches what CI installs. We're interested in reducing our dependencies if simple options exist. Got an idea? Try it out, we'd love to hear of your successes and failures! @@ -154,7 +83,6 @@ cargo check make check # Validate your code actually does compile (in dev mode) cargo build -make build-dev # Validate your test pass cargo test sources::example make test SCOPE="sources::example" diff --git a/rfcs/2020-05-25-2685-dev-workflow-simplification.md b/rfcs/2020-05-25-2685-dev-workflow-simplification.md index 99a186c843f0c..ae50463376f76 100644 --- a/rfcs/2020-05-25-2685-dev-workflow-simplification.md +++ b/rfcs/2020-05-25-2685-dev-workflow-simplification.md @@ -1,5 +1,7 @@ # RFC 2685 - 2020-05-28 - Dev Workflow Simplification +> **Archived:** This RFC is kept for historical reference only. The Docker/Podman dev environment it proposed (`make environment`, `ENVIRONMENT=true`, the `timberio/vector-dev` image) is no longer maintained and the supporting Makefile targets, workflow, and image have been removed. For current contributor instructions, see [CONTRIBUTING.md](../CONTRIBUTING.md) and [docs/DEVELOPING.md](../docs/DEVELOPING.md). + Vector's `Makefile` serves a variety of purposes, and this RFC attempts to tame the complexity of common dev tasks, improving contributor and team member experience. It proposes a practical base `environment` container that merges the functionality of our non-integration test containers into one. It then suggests making common dev `make` tasks to rely on the caller environment having all dependencies, done at the same time it suggests adding `make` tasks to run common `make` tasks inside the environment. Finally, it suggests updating documentation to suggest users can use their native toolchains, `docker`. diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile deleted file mode 100644 index 4502f2f71ddba..0000000000000 --- a/scripts/environment/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM docker.io/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 -ENV DEBIAN_FRONTEND=noninteractive \ - TZ='America/New York' \ - PATH=/root/.cargo/bin:/root/.local/bin/:$PATH \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 \ - CROSS_DOCKER_IN_DOCKER=true - -# Container junk -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get install -y git - -WORKDIR /git/vectordotdev/vector - -# Setup the env -COPY scripts/environment/*.sh scripts/environment/ -COPY scripts/environment/npm-tools/ scripts/environment/npm-tools/ -RUN ./scripts/environment/bootstrap-ubuntu-24.04.sh - -# Setup the toolchain -COPY rust-toolchain.toml \ - /git/vectordotdev/vector/ -RUN ./scripts/environment/prepare.sh && ./scripts/environment/setup-helm.sh - -# Declare volumes -VOLUME /vector -VOLUME /vector/target -VOLUME /root/.cargo -VOLUME /root/.rustup - -# Prepare for use -COPY ./scripts/environment/entrypoint.sh / -ENTRYPOINT [ "/entrypoint.sh" ] -CMD [ "bash" ] diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh deleted file mode 100755 index 6cc2c010157d6..0000000000000 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Refer to https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md -# for all runner information such as OS version and installed software. - -set -e -o verbose - -if [ -n "$RUSTFLAGS" ] -then - # shellcheck disable=SC2016 - echo '$RUSTFLAGS MUST NOT be set in CI configs as it overrides settings in `.cargo/config.toml`.' - exit 1 -fi - -export DEBIAN_FRONTEND=noninteractive -export ACCEPT_EULA=Y - -# Configure apt for speed and efficiency -cat > /etc/apt/apt.conf.d/90-vector-optimizations <> "${GITHUB_PATH}" - # we often run into OOM issues in CI due to the low memory vs. CPU ratio on c5 instances - echo "CARGO_BUILD_JOBS=$(($(nproc) /2))" >> "${GITHUB_ENV}" -else - echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> "${HOME}/.bash_profile" -fi - -# Add repositories for Docker and Node.js first, then do a single update -NEED_UPDATE=0 - -if ! [ -x "$(command -v docker)" ]; then - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu \ - xenial \ - stable" - NEED_UPDATE=1 -fi - -if ! [ -x "$(command -v node)" ]; then - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://deb.nodesource.com/node_18.x \ - nodistro \ - main" - NEED_UPDATE=1 -fi - -# Only update if we added new repositories -if [ $NEED_UPDATE -eq 1 ]; then - apt-get update --yes -fi - -# Install Docker if needed -if ! [ -x "$(command -v docker)" ]; then - apt-get install --yes docker-ce docker-ce-cli containerd.io - # ubuntu user doesn't exist in scripts/environment/Dockerfile which runs this - usermod --append --groups docker ubuntu || true -fi - -bash scripts/environment/install-protoc.sh - -# Install Node.js if needed -if ! [ -x "$(command -v node)" ]; then - apt-get install --yes nodejs - # enable corepack (enables the yarn and pnpm package managers) - # ref: https://nodejs.org/docs/latest-v18.x/api/corepack.html - corepack enable -fi - -# Apt cleanup -apt-get clean - -# Set up the default "deny all warnings" build flags -CARGO_OVERRIDE_DIR="${HOME}/.cargo" -CARGO_OVERRIDE_CONF="${CARGO_OVERRIDE_DIR}/config.toml" -cat <>"$CARGO_OVERRIDE_CONF" -[target.'cfg(linux)'] -rustflags = [ "-D", "warnings" ] -EOF - -# Install mold, because the system linker wastes a bunch of time. -# -# Notably, we don't install/configure it when we're going to do anything with `cross`, as `cross` takes the Cargo -# configuration from the host system and ships it over... which isn't good when we're overriding the `rustc-wrapper` -# and all of that. -if [ -z "${DISABLE_MOLD:-""}" ] ; then - # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory - # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. - TEMP=$(mktemp -d) - MOLD_VERSION=2.40.4 - MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux - curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ - --output "$TEMP/${MOLD_TARGET}.tar.gz" - tar \ - -xvf "${TEMP}/${MOLD_TARGET}.tar.gz" \ - -C "${TEMP}" - cp "${TEMP}/${MOLD_TARGET}/bin/mold" /usr/bin/mold - cp "${TEMP}/${MOLD_TARGET}/lib/mold/mold-wrapper.so" /usr/bin/mold-wrapper.so - rm -rf "$TEMP" - - # Create our rustc wrapper script that we'll use to actually invoke `rustc` such that `mold` will wrap it and intercept - # anything linking calls to use `mold` instead of `ld`, etc. - CARGO_BIN_DIR="${CARGO_OVERRIDE_DIR}/bin" - mkdir -p "$CARGO_BIN_DIR" - - RUSTC_WRAPPER="${CARGO_BIN_DIR}/wrap-rustc" - cat <"$RUSTC_WRAPPER" -#!/bin/sh -exec mold -run "\$@" -EOF - chmod +x "$RUSTC_WRAPPER" - - # Now configure Cargo to use our rustc wrapper script. - { - echo "[build]" - echo "rustc-wrapper = \"${RUSTC_WRAPPER}\"" - } >> "$CARGO_OVERRIDE_CONF" -fi - -mkdir -p /var/lib/vector -chmod 777 /var/lib/vector diff --git a/scripts/environment/entrypoint.sh b/scripts/environment/entrypoint.sh deleted file mode 100755 index b97484e991fc6..0000000000000 --- a/scripts/environment/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# set HOSTNAME to container id for `cross` -if [ -f /.docker-container-id ]; then - HOSTNAME="$(cat /.docker-container-id)" - export HOSTNAME -fi - -if [ -z "$HOSTNAME" ]; then - echo "Failed to properly set HOSTNAME, cross may not work" - # Fallback if everything else fails - HOSTNAME="vector-environment" - export HOSTNAME -fi - -exec "$@" diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 6371db0a32c1d..41c9d6de5fbb5 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -147,7 +147,24 @@ maybe_install_cargo_tool() { fi if ! $version_cmd --version 2>/dev/null | grep -q "^${version_pattern}"; then - cargo "${installer[@]}" "$tool" --version "$version" --force --locked + local should_install=true + # Outside CI, preserve a newer-than-pin version the user already has. + # `cargo install --force` would otherwise silently downgrade them. + if [[ -z "${CI:-}" ]]; then + local current + current=$($version_cmd --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + if [[ -n "$current" ]] && [[ "$current" != "$version" ]]; then + local newest + newest=$(printf '%s\n%s\n' "$current" "$version" | sort -V | tail -1) + if [[ "$newest" == "$current" ]]; then + echo "Keeping ${tool} ${current} (newer than pin ${version}). Set CI=1 to force the pin." + should_install=false + fi + fi + fi + if [[ "$should_install" == "true" ]]; then + cargo "${installer[@]}" "$tool" --version "$version" --force --locked + fi fi # cargo-llvm-cov requires the llvm-tools-preview rustup component @@ -199,6 +216,17 @@ maybe_install_npm_tools() { npm ci --prefix "${npm_tools_dir}" + # Outside CI, skip the global symlink to avoid a sudo write to /usr/local/bin + # (or equivalent). The Makefile prepends this directory to PATH, so `make` + # recipes find the tools automatically. + if [[ -z "${CI:-}" ]]; then + echo "npm tools installed under ${npm_tools_dir}/node_modules/.bin" + echo "Make recipes discover them automatically. To invoke directly from a" + echo "shell, add the directory to your PATH:" + echo " export PATH=\"${npm_tools_dir}/node_modules/.bin:\$PATH\"" + return 0 + fi + # Use sudo only when the target directory is not writable (e.g. /usr/local/bin # on Linux CI runners is root-owned, but Homebrew dirs on macOS are user-owned). local ln_cmd=(ln -sf) @@ -210,8 +238,12 @@ maybe_install_npm_tools() { done } -# Always ensure git safe.directory is set -git config --global --add safe.directory "$(pwd)" +# Set git safe.directory in CI where the repo may be checked out by a different +# uid than the user running git. Skipped on workstations: the contributor owns +# the checkout and a global config write is unnecessary. +if [[ -n "${CI:-}" ]]; then + git config --global --add safe.directory "$(pwd)" +fi REQUIRES_RUSTUP=(dd-rust-license-tool cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) REQUIRES_BINSTALL=(cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) diff --git a/scripts/environment/setup-helm.sh b/scripts/environment/setup-helm.sh deleted file mode 100755 index a16b731957b3c..0000000000000 --- a/scripts/environment/setup-helm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -KUBERNETES_VERSION="v1.18.6" -HELM_VERSION="v3.2.4" - -curl -Lo kubectl \ - "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" -sudo install kubectl /usr/local/bin/ && rm kubectl - -curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ - | tar -xzv --strip-components=1 --occurrence linux-amd64/helm -sudo install helm /usr/local/bin/ && rm helm - -curl -L "https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz" \ - | tar -xzv -sudo install kubeval /usr/local/bin/ && rm kubeval && rm README.md && rm LICENSE