From d7be39080b862d70975a7acf3c2d7f8486fc3c4c Mon Sep 17 00:00:00 2001 From: beechesII Date: Tue, 4 Jan 2022 15:47:33 +0100 Subject: [PATCH 01/26] added functionality to install vector from repository --- README.md | 34 ++++++++++++++++- galaxy.yml | 2 + roles/vector/defaults/main.yml | 7 ++-- roles/vector/handlers/main.yml | 2 +- roles/vector/tasks/config.yml | 29 ++++++++++++++ roles/vector/tasks/install.yml | 46 ++++++++++++++++++++++ roles/vector/tasks/main.yml | 70 ++++------------------------------ roles/vector/tasks/repo.yml | 25 ++++++++++++ 8 files changed, 147 insertions(+), 68 deletions(-) create mode 100644 roles/vector/tasks/config.yml create mode 100644 roles/vector/tasks/install.yml create mode 100644 roles/vector/tasks/repo.yml diff --git a/README.md b/README.md index 035f4c3..7df5d75 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,36 @@ This is an ansible role to set up [vector](https://vector.dev). It translates the YAML configuration to TOML, so any configuration is possible. -For available variables check out [defaults](roles/vector/defaults/main.yml) - Currently only amd64, arch64, arch7 through deb and rpm packages are supported + +## Variables + +| Variable | Required | Default | Description +|--------------------------------------------|----------|------------------------|------------ +| vector_template | yes | vector.toml.j2 | path of your vector.toml template +| vector_config_file | yes | /etc/vector/vector.toml | system path of your vector.toml configuration +| vector_nightly | no | false | use vector nightly build +| add_vector_docker_group | no | false | add user vector to group docker +| add_vector_journal_group | no | false | add user vector to group systemd-journal +| add_vector_journal_group | no | false | add user vector to group systemd-journal +| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories + +## Example for toml configuration in yaml +```yaml +sources: + journald: + type: journald + current_boot_only: true + +transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' +sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" +``` diff --git a/galaxy.yml b/galaxy.yml index ee8ed07..a8ead4f 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -7,6 +7,8 @@ authors: - Dimitris Zervas - Gaëtan Duchaussois - Dylan Humphreys + - Daniel Uhlmann + - Christopher Grau description: vector.dev ansible role license_file: LICENSE tags: diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 534df01..5e41761 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -1,9 +1,10 @@ vector_template: vector.toml.j2 vector_config_file: /etc/vector/vector.toml -vector_nightly: no +vector_nightly: false vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" -add_vector_docker_group: no # Add vector user to "docker" group -add_vector_journal_group: no # Add vector user to "systemd-journal" group +add_vector_docker_group: false +add_vector_journal_group: false +vector_install_from_repo: false sources: journald: diff --git a/roles/vector/handlers/main.yml b/roles/vector/handlers/main.yml index 9081f3d..4de2cc3 100644 --- a/roles/vector/handlers/main.yml +++ b/roles/vector/handlers/main.yml @@ -1,5 +1,5 @@ - name: restart vector service: state: restarted - daemon_reload: yes + daemon_reload: true name: vector diff --git a/roles/vector/tasks/config.yml b/roles/vector/tasks/config.yml new file mode 100644 index 0000000..520e985 --- /dev/null +++ b/roles/vector/tasks/config.yml @@ -0,0 +1,29 @@ +--- +- name: Copy config + template: + src: "{{ vector_template }}" + dest: "{{ vector_config_file }}" + mode: 0644 + notify: restart vector + +- name: Add vector user to docker group + user: + name: vector + groups: docker + append: true + when: add_vector_docker_group | bool + notify: restart vector + +- name: Add vector user to systemd-journal group + user: + name: vector + groups: systemd-journal + append: true + when: add_vector_journal_group | bool + notify: restart vector + +- name: Start vector + service: + state: started + enabled: true + name: vector diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml new file mode 100644 index 0000000..191ff5f --- /dev/null +++ b/roles/vector/tasks/install.yml @@ -0,0 +1,46 @@ +--- +- block: + # Workaround for latest version being named against a version number + - name: Get latest version + uri: + url: https://s3.amazonaws.com/packages.timber.io/?prefix=vector/latest&max-keys=1 + return_content: true + register: bucket_content + when: vector_version == "latest" + + - name: Set latest version + set_fact: + vector_version: "{{ bucket_content.content | regex_replace('.*\\n.*[^-]+-(?P[\\d\\.]+)-.*','\\g') }}" + when: vector_version == "latest" + + - name: Install Vector (Debian) + apt: + deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb" + install_recommends: true + notify: + - restart vector + vars: + version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" + arch: "{{ vector_debian_arch[ansible_machine] }}" + when: ansible_os_family == 'Debian' + + - name: Install Vector (RedHat) + yum: + name: "https://packages.timber.io/vector/{{ version }}/vector-{{ package_version }}.{{ arch }}.rpm" + state: present + disable_gpg_check: true # package is not signed + notify: + - restart vector + vars: + version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" + package_version: "{{ vector_version is match('latest') | ternary(vector_version, vector_version ~ '-1') }}" + arch: "{{ vector_redhat_arch[ansible_machine] }}" + when: ansible_os_family == 'RedHat' + when: vector_install_from_repo == false + +- block: + - name: Install vector + package: + name: vector + state: present + when: vector_install_from_repo == true diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index 81dbd93..0f7ceb9 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -1,65 +1,11 @@ -# Workaround for latest version being named against a version number -- name: Get latest version - uri: - url: https://s3.amazonaws.com/packages.timber.io/?prefix=vector/latest&max-keys=1 - return_content: true - register: bucket_content - when: vector_version == "latest" +--- -- name: Set latest version - set_fact: - vector_version: "{{ bucket_content.content | regex_replace('.*\\n.*[^-]+-(?P[\\d\\.]+)-.*','\\g') }}" - when: vector_version == "latest" +- name: Create repos for debian or redhat based systems + include_tasks: repo.yml + when: vector_install_from_repo | bool -- name: Install Vector (Debian) - apt: - deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb" - install_recommends: yes - notify: - - restart vector - vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - arch: "{{ vector_debian_arch[ansible_machine] }}" - when: ansible_os_family == 'Debian' +- name: Install vector + include_tasks: install.yml -- name: Install Vector (RedHat) - yum: - name: "https://packages.timber.io/vector/{{ version }}/vector-{{ package_version }}.{{ arch }}.rpm" - state: present - disable_gpg_check: yes # package is not signed - notify: - - restart vector - vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - package_version: "{{ vector_version is match('latest') | ternary(vector_version, vector_version ~ '-1') }}" - arch: "{{ vector_redhat_arch[ansible_machine] }}" - when: ansible_os_family == 'RedHat' - -- name: Copy config - template: - src: "{{ vector_template }}" - dest: "{{ vector_config_file }}" - mode: 0644 - notify: restart vector - -- name: Add vector user to docker group - user: - name: vector - groups: docker - append: yes - when: add_vector_docker_group | default(no) - notify: restart vector - -- name: Add vector user to systemd-journal group - user: - name: vector - groups: systemd-journal - append: yes - when: add_vector_journal_group | default(no) - notify: restart vector - -- name: Start Vector - service: - state: started - enabled: yes - name: vector +- name: Create configuration for vector + include_tasks: config.yml diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml new file mode 100644 index 0000000..aa0c277 --- /dev/null +++ b/roles/vector/tasks/repo.yml @@ -0,0 +1,25 @@ +--- + +- block: + - name: Add vector gpg key to apt + become: true + apt_key: + url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + state: present + + - name: Add vector repository for Debian based distributions + become: true + apt_repository: + repo: "deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main" + state: present + filename: timber-vector + when: ansible_os_family == "Debian" + +- name: Add vector repository for RedHat based distributions + yum_repository: + name: timber-vector + description: "Official repository for Timber / vector" + baseurl: https://download.fedoraproject.org/pub/epel/$releasever/$basearch/ + gpgcheck: true + gpgkey: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + when: ansible_os_family == "RedHat" From e4beb5a4e34c6f2d091e57d1164c64dc5980eb45 Mon Sep 17 00:00:00 2001 From: beechesII Date: Tue, 4 Jan 2022 15:53:36 +0100 Subject: [PATCH 02/26] changed baseurl for yum repository --- roles/vector/tasks/repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index aa0c277..d869966 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -19,7 +19,7 @@ yum_repository: name: timber-vector description: "Official repository for Timber / vector" - baseurl: https://download.fedoraproject.org/pub/epel/$releasever/$basearch/ + baseurl: https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch gpgcheck: true gpgkey: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key when: ansible_os_family == "RedHat" From 5f2c46284352b3a43ba59addcf60c44259c9cec8 Mon Sep 17 00:00:00 2001 From: beechesII Date: Tue, 11 Jan 2022 13:05:53 +0100 Subject: [PATCH 03/26] moved sources,transforms and sinks from defaults to converge.yml updated README.md --- README.md | 30 +++++++++++++++++++++- roles/vector/defaults/main.yml | 17 ------------ roles/vector/molecule/default/converge.yml | 16 ++++++++++++ 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7df5d75..8b44895 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,11 @@ Currently only amd64, arch64, arch7 through deb and rpm packages are supported | add_vector_journal_group | no | false | add user vector to group systemd-journal | add_vector_journal_group | no | false | add user vector to group systemd-journal | vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories +| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) +| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) +| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) -## Example for toml configuration in yaml +## Example for configuration with ansible ```yaml sources: journald: @@ -32,9 +35,34 @@ transforms: inputs: - journald pattern: '(?\\d+)%{GREEDYDATA}' + sinks: vector: type: vector inputs: ["journald"] address: "vector.example.com:9000" ``` + +## Example playbook +```yaml +- name: install and configure vector + hosts: all + vars: + sources: + journald: + type: journald + current_boot_only: true + transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' + sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" + roles: + - vector +``` diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 5e41761..d1c7a41 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -5,20 +5,3 @@ vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" add_vector_docker_group: false add_vector_journal_group: false vector_install_from_repo: false - -sources: - journald: - type: journald - current_boot_only: true - -transforms: - grok: - type: grok_parser - inputs: - - journald - pattern: '(?\\d+)%{GREEDYDATA}' -sinks: - vector: - type: vector - inputs: ["journald"] - address: "vector.example.com:9000" diff --git a/roles/vector/molecule/default/converge.yml b/roles/vector/molecule/default/converge.yml index 6d39e2e..e59ddbb 100644 --- a/roles/vector/molecule/default/converge.yml +++ b/roles/vector/molecule/default/converge.yml @@ -1,6 +1,22 @@ --- - name: Converge hosts: all + vars: + sources: + journald: + type: journald + current_boot_only: true + transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' + sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" tasks: - name: "Include vector" include_role: From 39c971980b48d37bbe44e7dd1167f760771cda52 Mon Sep 17 00:00:00 2001 From: beechesII Date: Fri, 4 Feb 2022 15:32:38 +0100 Subject: [PATCH 04/26] remove duplicate entry in readme.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8b44895..5d03d37 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Currently only amd64, arch64, arch7 through deb and rpm packages are supported | vector_nightly | no | false | use vector nightly build | add_vector_docker_group | no | false | add user vector to group docker | add_vector_journal_group | no | false | add user vector to group systemd-journal -| add_vector_journal_group | no | false | add user vector to group systemd-journal | vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories | sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) | transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) From 44330fba8e2626254d512301f12e82ba8b582a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wielgosz?= Date: Thu, 28 Apr 2022 14:01:35 +0200 Subject: [PATCH 05/26] Fix deb package url --- roles/vector/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index 81dbd93..8eb0594 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -13,7 +13,7 @@ - name: Install Vector (Debian) apt: - deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb" + deb: "https://packages.timber.io/vector/{{ version }}/vector_{{ version }}-1_{{ arch }}.deb" install_recommends: yes notify: - restart vector From 5a46e3b34da8830862473c4791bbaf5a1cbc7312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Duchaussois?= Date: Wed, 8 Jun 2022 22:13:34 +0200 Subject: [PATCH 06/26] add files required by ansible-galaxy --- galaxy.yml | 1 + meta/runtime.yml | 1 + 2 files changed, 2 insertions(+) create mode 100644 meta/runtime.yml diff --git a/galaxy.yml b/galaxy.yml index ee8ed07..1fcd17d 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -16,3 +16,4 @@ tags: repository: https://github.com/dzervas/ansible-vector documentation: https://github.com/dzervas/ansible-vector issues: https://github.com/dzervas/ansible-vector/issues +version: 0.1.5 diff --git a/meta/runtime.yml b/meta/runtime.yml new file mode 100644 index 0000000..938ca97 --- /dev/null +++ b/meta/runtime.yml @@ -0,0 +1 @@ +requires_ansible: '>=2.9.10' From f0019c7a547db6b9ca67db0867413972524b368a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Duchaussois?= Date: Thu, 9 Jun 2022 14:27:27 +0200 Subject: [PATCH 07/26] fix tests --- roles/vector/defaults/main.yml | 10 +++++----- roles/vector/molecule/default/molecule.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 534df01..efbbb96 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -9,7 +9,6 @@ sources: journald: type: journald current_boot_only: true - transforms: grok: type: grok_parser @@ -17,7 +16,8 @@ transforms: - journald pattern: '(?\\d+)%{GREEDYDATA}' sinks: - vector: - type: vector - inputs: ["journald"] - address: "vector.example.com:9000" + console: + type: console + encoding.codec: json + inputs: + - grok diff --git a/roles/vector/molecule/default/molecule.yml b/roles/vector/molecule/default/molecule.yml index 38186a5..f8fa61a 100644 --- a/roles/vector/molecule/default/molecule.yml +++ b/roles/vector/molecule/default/molecule.yml @@ -45,8 +45,8 @@ platforms: - /tmp volumes: - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: centos8 - image: jrei/systemd-centos:8 + - name: rocky8 + image: rockylinux:8 privileged: true command: /usr/sbin/init tmpfs: From ca065269cea10b9a7ab652805eab547b84add2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Duchaussois?= Date: Fri, 10 Jun 2022 15:20:01 +0200 Subject: [PATCH 08/26] Add a template for yaml config generation and ensure vector use the config file --- roles/vector/tasks/main.yml | 6 ++++++ roles/vector/templates/vector.yaml.j2 | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 roles/vector/templates/vector.yaml.j2 diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index 8eb0594..3c2d233 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -42,6 +42,12 @@ mode: 0644 notify: restart vector +- name: Use the right config_file + lineinfile: + path: /etc/default/vector + line: "VECTOR_CONFIG={{ vector_config_file }}" + state: present + - name: Add vector user to docker group user: name: vector diff --git a/roles/vector/templates/vector.yaml.j2 b/roles/vector/templates/vector.yaml.j2 new file mode 100644 index 0000000..6da1ff3 --- /dev/null +++ b/roles/vector/templates/vector.yaml.j2 @@ -0,0 +1,7 @@ +{{ vector_general_config | default("") }} +sources: + {{ sources | to_nice_yaml | indent(2) }} +transforms: + {{ transforms | to_nice_yaml | indent(2) }} +sinks: + {{ sinks | to_nice_yaml | indent(2) }} From c2c412b1588cbfbf006912f44b0cc94d2477438e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Duchaussois?= Date: Fri, 10 Jun 2022 15:20:26 +0200 Subject: [PATCH 09/26] Add test for config file usage --- .github/workflows/push.yml | 2 +- roles/vector/molecule/features/converge.yml | 33 +++++++++++++++++++++ roles/vector/molecule/features/molecule.yml | 33 +++++++++++++++++++++ roles/vector/molecule/features/verify.yml | 33 +++++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 roles/vector/molecule/features/converge.yml create mode 100644 roles/vector/molecule/features/molecule.yml create mode 100644 roles/vector/molecule/features/verify.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3da9bf4..182258c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -28,6 +28,6 @@ jobs: ansible-lint --version - name: Test run: | - cd roles/vector && molecule test + cd roles/vector && molecule test --all diff --git a/roles/vector/molecule/features/converge.yml b/roles/vector/molecule/features/converge.yml new file mode 100644 index 0000000..f1926d7 --- /dev/null +++ b/roles/vector/molecule/features/converge.yml @@ -0,0 +1,33 @@ +--- +- name: Converge + hosts: all + tasks: + - name: "Include vector" + include_role: + name: "vector" + vars: + vector_template: vector.yaml.j2 + vector_config_file: /etc/vector/vector.yaml + sources: + journald: + type: journald + current_boot_only: true + tcp: + type: socket + mode: tcp + address: 0.0.0.0:6000 + encoding: + codec: json + transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' + sinks: + console: + type: console + encoding: + codec: json + inputs: + - grok diff --git a/roles/vector/molecule/features/molecule.yml b/roles/vector/molecule/features/molecule.yml new file mode 100644 index 0000000..14d5079 --- /dev/null +++ b/roles/vector/molecule/features/molecule.yml @@ -0,0 +1,33 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: | + set -e + yamllint . + ansible-lint + flake8 +platforms: + - name: debian10 + image: jrei/systemd-debian:10 + privileged: true + command: /lib/systemd/systemd + tmpfs: + - /run + - /tmp + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - name: rocky8 + image: rockylinux:8 + privileged: true + command: /usr/sbin/init + tmpfs: + - /run + - /tmp + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro +provisioner: + name: ansible +verifier: + name: ansible diff --git a/roles/vector/molecule/features/verify.yml b/roles/vector/molecule/features/verify.yml new file mode 100644 index 0000000..8b8fffe --- /dev/null +++ b/roles/vector/molecule/features/verify.yml @@ -0,0 +1,33 @@ +--- +- name: Verify + hosts: all + gather_facts: true + tasks: + - name: Install netstat + yum: + name: + - net-tools + - procps + state: present + when: ansible_os_family == 'RedHat' + - name: Install netstat + apt: + name: + - net-tools + - procps + state: present + update_cache: yes + when: ansible_os_family == 'Debian' + - name: Execute vector + command: vector --version + changed_when: false + register: vector_version_rc + - name: Check port 6000 is open + community.general.listen_ports_facts: + - name: Assert vector is installed + assert: + that: + - vector_version_rc is success + - 6000 in open_ports + vars: + open_ports: "{{ ansible_facts.tcp_listen | map(attribute='port') }}" From 97966fde58561970f9b23113c92289d457517ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan?= <36162164+gaetanfl@users.noreply.github.com> Date: Fri, 10 Jun 2022 18:05:53 +0200 Subject: [PATCH 10/26] Prepare release --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 1fcd17d..cff1186 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -16,4 +16,4 @@ tags: repository: https://github.com/dzervas/ansible-vector documentation: https://github.com/dzervas/ansible-vector issues: https://github.com/dzervas/ansible-vector/issues -version: 0.1.5 +version: 0.1.6 From 6265224591d893773a439ec45e4f07846c1876c5 Mon Sep 17 00:00:00 2001 From: beechesII Date: Tue, 4 Jan 2022 15:47:33 +0100 Subject: [PATCH 11/26] added functionality to install vector from repository --- README.md | 34 ++++++++++++++- galaxy.yml | 2 + roles/vector/defaults/main.yml | 7 ++-- roles/vector/handlers/main.yml | 2 +- roles/vector/tasks/config.yml | 29 +++++++++++++ roles/vector/tasks/install.yml | 46 ++++++++++++++++++++ roles/vector/tasks/main.yml | 76 ++++------------------------------ roles/vector/tasks/repo.yml | 25 +++++++++++ 8 files changed, 147 insertions(+), 74 deletions(-) create mode 100644 roles/vector/tasks/config.yml create mode 100644 roles/vector/tasks/install.yml create mode 100644 roles/vector/tasks/repo.yml diff --git a/README.md b/README.md index 035f4c3..7df5d75 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,36 @@ This is an ansible role to set up [vector](https://vector.dev). It translates the YAML configuration to TOML, so any configuration is possible. -For available variables check out [defaults](roles/vector/defaults/main.yml) - Currently only amd64, arch64, arch7 through deb and rpm packages are supported + +## Variables + +| Variable | Required | Default | Description +|--------------------------------------------|----------|------------------------|------------ +| vector_template | yes | vector.toml.j2 | path of your vector.toml template +| vector_config_file | yes | /etc/vector/vector.toml | system path of your vector.toml configuration +| vector_nightly | no | false | use vector nightly build +| add_vector_docker_group | no | false | add user vector to group docker +| add_vector_journal_group | no | false | add user vector to group systemd-journal +| add_vector_journal_group | no | false | add user vector to group systemd-journal +| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories + +## Example for toml configuration in yaml +```yaml +sources: + journald: + type: journald + current_boot_only: true + +transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' +sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" +``` diff --git a/galaxy.yml b/galaxy.yml index cff1186..86bf2eb 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -7,6 +7,8 @@ authors: - Dimitris Zervas - Gaëtan Duchaussois - Dylan Humphreys + - Daniel Uhlmann + - Christopher Grau description: vector.dev ansible role license_file: LICENSE tags: diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index efbbb96..7da9d32 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -1,9 +1,10 @@ vector_template: vector.toml.j2 vector_config_file: /etc/vector/vector.toml -vector_nightly: no +vector_nightly: false vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" -add_vector_docker_group: no # Add vector user to "docker" group -add_vector_journal_group: no # Add vector user to "systemd-journal" group +add_vector_docker_group: false +add_vector_journal_group: false +vector_install_from_repo: false sources: journald: diff --git a/roles/vector/handlers/main.yml b/roles/vector/handlers/main.yml index 9081f3d..4de2cc3 100644 --- a/roles/vector/handlers/main.yml +++ b/roles/vector/handlers/main.yml @@ -1,5 +1,5 @@ - name: restart vector service: state: restarted - daemon_reload: yes + daemon_reload: true name: vector diff --git a/roles/vector/tasks/config.yml b/roles/vector/tasks/config.yml new file mode 100644 index 0000000..520e985 --- /dev/null +++ b/roles/vector/tasks/config.yml @@ -0,0 +1,29 @@ +--- +- name: Copy config + template: + src: "{{ vector_template }}" + dest: "{{ vector_config_file }}" + mode: 0644 + notify: restart vector + +- name: Add vector user to docker group + user: + name: vector + groups: docker + append: true + when: add_vector_docker_group | bool + notify: restart vector + +- name: Add vector user to systemd-journal group + user: + name: vector + groups: systemd-journal + append: true + when: add_vector_journal_group | bool + notify: restart vector + +- name: Start vector + service: + state: started + enabled: true + name: vector diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml new file mode 100644 index 0000000..191ff5f --- /dev/null +++ b/roles/vector/tasks/install.yml @@ -0,0 +1,46 @@ +--- +- block: + # Workaround for latest version being named against a version number + - name: Get latest version + uri: + url: https://s3.amazonaws.com/packages.timber.io/?prefix=vector/latest&max-keys=1 + return_content: true + register: bucket_content + when: vector_version == "latest" + + - name: Set latest version + set_fact: + vector_version: "{{ bucket_content.content | regex_replace('.*\\n.*[^-]+-(?P[\\d\\.]+)-.*','\\g') }}" + when: vector_version == "latest" + + - name: Install Vector (Debian) + apt: + deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb" + install_recommends: true + notify: + - restart vector + vars: + version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" + arch: "{{ vector_debian_arch[ansible_machine] }}" + when: ansible_os_family == 'Debian' + + - name: Install Vector (RedHat) + yum: + name: "https://packages.timber.io/vector/{{ version }}/vector-{{ package_version }}.{{ arch }}.rpm" + state: present + disable_gpg_check: true # package is not signed + notify: + - restart vector + vars: + version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" + package_version: "{{ vector_version is match('latest') | ternary(vector_version, vector_version ~ '-1') }}" + arch: "{{ vector_redhat_arch[ansible_machine] }}" + when: ansible_os_family == 'RedHat' + when: vector_install_from_repo == false + +- block: + - name: Install vector + package: + name: vector + state: present + when: vector_install_from_repo == true diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index 3c2d233..0f7ceb9 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -1,71 +1,11 @@ -# Workaround for latest version being named against a version number -- name: Get latest version - uri: - url: https://s3.amazonaws.com/packages.timber.io/?prefix=vector/latest&max-keys=1 - return_content: true - register: bucket_content - when: vector_version == "latest" +--- -- name: Set latest version - set_fact: - vector_version: "{{ bucket_content.content | regex_replace('.*\\n.*[^-]+-(?P[\\d\\.]+)-.*','\\g') }}" - when: vector_version == "latest" +- name: Create repos for debian or redhat based systems + include_tasks: repo.yml + when: vector_install_from_repo | bool -- name: Install Vector (Debian) - apt: - deb: "https://packages.timber.io/vector/{{ version }}/vector_{{ version }}-1_{{ arch }}.deb" - install_recommends: yes - notify: - - restart vector - vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - arch: "{{ vector_debian_arch[ansible_machine] }}" - when: ansible_os_family == 'Debian' +- name: Install vector + include_tasks: install.yml -- name: Install Vector (RedHat) - yum: - name: "https://packages.timber.io/vector/{{ version }}/vector-{{ package_version }}.{{ arch }}.rpm" - state: present - disable_gpg_check: yes # package is not signed - notify: - - restart vector - vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - package_version: "{{ vector_version is match('latest') | ternary(vector_version, vector_version ~ '-1') }}" - arch: "{{ vector_redhat_arch[ansible_machine] }}" - when: ansible_os_family == 'RedHat' - -- name: Copy config - template: - src: "{{ vector_template }}" - dest: "{{ vector_config_file }}" - mode: 0644 - notify: restart vector - -- name: Use the right config_file - lineinfile: - path: /etc/default/vector - line: "VECTOR_CONFIG={{ vector_config_file }}" - state: present - -- name: Add vector user to docker group - user: - name: vector - groups: docker - append: yes - when: add_vector_docker_group | default(no) - notify: restart vector - -- name: Add vector user to systemd-journal group - user: - name: vector - groups: systemd-journal - append: yes - when: add_vector_journal_group | default(no) - notify: restart vector - -- name: Start Vector - service: - state: started - enabled: yes - name: vector +- name: Create configuration for vector + include_tasks: config.yml diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml new file mode 100644 index 0000000..aa0c277 --- /dev/null +++ b/roles/vector/tasks/repo.yml @@ -0,0 +1,25 @@ +--- + +- block: + - name: Add vector gpg key to apt + become: true + apt_key: + url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + state: present + + - name: Add vector repository for Debian based distributions + become: true + apt_repository: + repo: "deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main" + state: present + filename: timber-vector + when: ansible_os_family == "Debian" + +- name: Add vector repository for RedHat based distributions + yum_repository: + name: timber-vector + description: "Official repository for Timber / vector" + baseurl: https://download.fedoraproject.org/pub/epel/$releasever/$basearch/ + gpgcheck: true + gpgkey: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + when: ansible_os_family == "RedHat" From 3daa0e80a06affeff495a67558167b6b66a88096 Mon Sep 17 00:00:00 2001 From: beechesII Date: Tue, 4 Jan 2022 15:53:36 +0100 Subject: [PATCH 12/26] changed baseurl for yum repository moved sources,transforms and sinks from defaults to converge.yml updated README.md remove duplicate entry in readme.md --- README.md | 31 ++++++++++++++++++++-- roles/vector/defaults/main.yml | 3 +++ roles/vector/molecule/default/converge.yml | 16 +++++++++++ roles/vector/tasks/repo.yml | 2 +- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7df5d75..5d03d37 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,12 @@ Currently only amd64, arch64, arch7 through deb and rpm packages are supported | vector_nightly | no | false | use vector nightly build | add_vector_docker_group | no | false | add user vector to group docker | add_vector_journal_group | no | false | add user vector to group systemd-journal -| add_vector_journal_group | no | false | add user vector to group systemd-journal | vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories +| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) +| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) +| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) -## Example for toml configuration in yaml +## Example for configuration with ansible ```yaml sources: journald: @@ -32,9 +34,34 @@ transforms: inputs: - journald pattern: '(?\\d+)%{GREEDYDATA}' + sinks: vector: type: vector inputs: ["journald"] address: "vector.example.com:9000" ``` + +## Example playbook +```yaml +- name: install and configure vector + hosts: all + vars: + sources: + journald: + type: journald + current_boot_only: true + transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' + sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" + roles: + - vector +``` diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 7da9d32..f0d66cf 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -5,6 +5,7 @@ vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" add_vector_docker_group: false add_vector_journal_group: false vector_install_from_repo: false +<<<<<<< HEAD sources: journald: @@ -22,3 +23,5 @@ sinks: encoding.codec: json inputs: - grok +======= +>>>>>>> 5f2c462 (moved sources,transforms and sinks from defaults to converge.yml) diff --git a/roles/vector/molecule/default/converge.yml b/roles/vector/molecule/default/converge.yml index 6d39e2e..e59ddbb 100644 --- a/roles/vector/molecule/default/converge.yml +++ b/roles/vector/molecule/default/converge.yml @@ -1,6 +1,22 @@ --- - name: Converge hosts: all + vars: + sources: + journald: + type: journald + current_boot_only: true + transforms: + grok: + type: grok_parser + inputs: + - journald + pattern: '(?\\d+)%{GREEDYDATA}' + sinks: + vector: + type: vector + inputs: ["journald"] + address: "vector.example.com:9000" tasks: - name: "Include vector" include_role: diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index aa0c277..d869966 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -19,7 +19,7 @@ yum_repository: name: timber-vector description: "Official repository for Timber / vector" - baseurl: https://download.fedoraproject.org/pub/epel/$releasever/$basearch/ + baseurl: https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch gpgcheck: true gpgkey: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key when: ansible_os_family == "RedHat" From 7db51f83564b441b6b82ff9406d708db65645e34 Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 31 Aug 2022 11:19:25 +0200 Subject: [PATCH 13/26] make repository configurable and vector version, in case user want to use a repo proxy and want to configure the installed vector version added variables vector_repo, vector_repo_key and vector_package defined defaults for new variables updated README.md --- README.md | 3 +++ roles/vector/defaults/main.yml | 12 +++++++++--- roles/vector/tasks/install.yml | 2 +- roles/vector/tasks/repo.yml | 8 ++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5d03d37..50344d3 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ Currently only amd64, arch64, arch7 through deb and rpm packages are supported | add_vector_docker_group | no | false | add user vector to group docker | add_vector_journal_group | no | false | add user vector to group systemd-journal | vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories +| vector_repo_key | no | `https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key` | configurable repo key, in case repo proxy is used +| vector_repo | no | Debian: `deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main`
Redhat: `https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch` | configurable repo, in case repo proxy is used +| vector_package | no | vector | option to define vector version with package name | sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) | transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) | sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 3f51c7e..d9f54aa 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -1,8 +1,14 @@ vector_template: vector.toml.j2 vector_config_file: /etc/vector/vector.toml -vector_nightly: false -vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" + add_vector_docker_group: false add_vector_journal_group: false -vector_install_from_repo: false +vector_nightly: false +vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" +vector_install_from_repo: false +vector_package: vector +vector_repo_key: "https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key" +vector_repo: + Debian: "deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main" + RedHat: "https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch" diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml index 191ff5f..964f025 100644 --- a/roles/vector/tasks/install.yml +++ b/roles/vector/tasks/install.yml @@ -41,6 +41,6 @@ - block: - name: Install vector package: - name: vector + name: "{{ vector_package }}" state: present when: vector_install_from_repo == true diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index d869966..eb52aec 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -4,13 +4,13 @@ - name: Add vector gpg key to apt become: true apt_key: - url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + url: "{{ vector_repo_key }}" state: present - name: Add vector repository for Debian based distributions become: true apt_repository: - repo: "deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main" + repo: "{{ vector_repo[ansible_os_family] }}" state: present filename: timber-vector when: ansible_os_family == "Debian" @@ -19,7 +19,7 @@ yum_repository: name: timber-vector description: "Official repository for Timber / vector" - baseurl: https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch + baseurl: "{{ vector_repo[ansible_os_family] }}" gpgcheck: true - gpgkey: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key + gpgkey: "{{ vector_repo_key }}" when: ansible_os_family == "RedHat" From c66112e16f4c9bde81eafcbb153a10a37ba43215 Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 31 Aug 2022 11:34:51 +0200 Subject: [PATCH 14/26] fix repo creation --- roles/vector/tasks/repo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index eb52aec..e35c05a 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -10,7 +10,7 @@ - name: Add vector repository for Debian based distributions become: true apt_repository: - repo: "{{ vector_repo[ansible_os_family] }}" + repo: "{{ vector_repo }}" state: present filename: timber-vector when: ansible_os_family == "Debian" @@ -19,7 +19,7 @@ yum_repository: name: timber-vector description: "Official repository for Timber / vector" - baseurl: "{{ vector_repo[ansible_os_family] }}" + baseurl: "{{ vector_repo }}" gpgcheck: true gpgkey: "{{ vector_repo_key }}" when: ansible_os_family == "RedHat" From 0c7759e3a302b983fdbc0e4cb632a5427340db27 Mon Sep 17 00:00:00 2001 From: Christopher Grau <66788631+beechesII@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:40:37 +0100 Subject: [PATCH 15/26] make transforms realy not required --- roles/vector/templates/vector.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/vector/templates/vector.yaml.j2 b/roles/vector/templates/vector.yaml.j2 index 6da1ff3..908d040 100644 --- a/roles/vector/templates/vector.yaml.j2 +++ b/roles/vector/templates/vector.yaml.j2 @@ -1,7 +1,9 @@ {{ vector_general_config | default("") }} sources: {{ sources | to_nice_yaml | indent(2) }} +{% if transforms is defined %} transforms: {{ transforms | to_nice_yaml | indent(2) }} +{% endif %} sinks: {{ sinks | to_nice_yaml | indent(2) }} From 2bb28eef5d1ef9a96700955e74dac1aa2fee60c3 Mon Sep 17 00:00:00 2001 From: Christopher Grau <66788631+beechesII@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:02:58 +0100 Subject: [PATCH 16/26] Update vector repository and lint project (#2) update vector repositories and lint project Co-authored-by: Christopher Grau Co-authored-by: sjedlick <104558780+sjedlick@users.noreply.github.com> --- .config/ansible-lint.yml | 9 +++ .github/workflows/ansible-lint.yml | 32 +++++++++ .github/workflows/molecule.yml | 70 +++++++++++++++++++ .github/workflows/new_version.yml | 24 ------- .github/workflows/push.yml | 33 --------- CHANGELOG.md | 3 + README.md | 48 +++++++------ galaxy.yml | 21 +++--- meta/main.yml | 17 +++++ meta/runtime.yml | 3 +- .../default => molecule/vector}/converge.yml | 15 ++-- molecule/vector/molecule.yml | 37 ++++++++++ molecule/vector/verify.yml | 31 ++++++++ requirements-tests.txt | 4 -- requirements.txt | 4 ++ roles/vector/.yamllint | 33 --------- roles/vector/README.md | 1 - roles/vector/defaults/main.yml | 33 ++++++--- roles/vector/handlers/main.yml | 5 +- roles/vector/meta/main.yml | 20 ------ roles/vector/molecule/default/molecule.yml | 61 ---------------- roles/vector/molecule/default/verify.yml | 20 ------ roles/vector/molecule/features/converge.yml | 33 --------- roles/vector/molecule/features/molecule.yml | 33 --------- roles/vector/molecule/features/verify.yml | 33 --------- roles/vector/tasks/config.yml | 31 ++++---- roles/vector/tasks/install.yml | 53 +++++--------- roles/vector/tasks/main.yml | 8 +-- roles/vector/tasks/repo.yml | 57 ++++++++++----- roles/vector/vars/main.yml | 8 --- 30 files changed, 352 insertions(+), 428 deletions(-) create mode 100644 .config/ansible-lint.yml create mode 100644 .github/workflows/ansible-lint.yml create mode 100644 .github/workflows/molecule.yml delete mode 100644 .github/workflows/new_version.yml delete mode 100644 .github/workflows/push.yml create mode 100644 CHANGELOG.md create mode 100644 meta/main.yml rename {roles/vector/molecule/default => molecule/vector}/converge.yml (54%) create mode 100644 molecule/vector/molecule.yml create mode 100644 molecule/vector/verify.yml delete mode 100644 requirements-tests.txt create mode 100644 requirements.txt delete mode 100644 roles/vector/.yamllint delete mode 120000 roles/vector/README.md delete mode 100644 roles/vector/meta/main.yml delete mode 100644 roles/vector/molecule/default/molecule.yml delete mode 100644 roles/vector/molecule/default/verify.yml delete mode 100644 roles/vector/molecule/features/converge.yml delete mode 100644 roles/vector/molecule/features/molecule.yml delete mode 100644 roles/vector/molecule/features/verify.yml delete mode 100644 roles/vector/vars/main.yml diff --git a/.config/ansible-lint.yml b/.config/ansible-lint.yml new file mode 100644 index 0000000..b5937a0 --- /dev/null +++ b/.config/ansible-lint.yml @@ -0,0 +1,9 @@ +--- +# .ansible-lint +exclude_paths: + - .cache/ # implicit unless exclude_paths is defined in config + - molecule/ + - .github/ + +# install collection dependencies +offline: false diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 0000000..71cdfe7 --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,32 @@ +--- +name: Ansible Lint # feel free to pick your own name + +on: # yamllint disable-line rule:truthy + # Run CI against all pushes (direct commits, also merged PRs), Pull Requests + push: + branches: [master] + paths: + - 'roles/**' + - 'molecule/**' + - 'requirements.txt' + - '.github/workflows/ansible-lint.yml' + - '.config/ansible-lint.yml' + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + paths: + - 'roles/**' + - 'molecule/**' + - 'requirements.txt' + - '.github/workflows/ansible-lint.yml' + - '.config/ansible-lint.yml' + +jobs: + ansible-lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Lint Ansible Playbook + uses: ansible/ansible-lint@44be233dbd6a8a6d8f3c5297c318ed4ed4644c32 # v24 diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml new file mode 100644 index 0000000..a8cfe89 --- /dev/null +++ b/.github/workflows/molecule.yml @@ -0,0 +1,70 @@ +--- +name: molecule +on: # yamllint disable-line rule:truthy + workflow_dispatch: + push: + branches: [master] + paths: + - 'roles/vector/**' + - 'molecule/ssh_hardening/**' + - '.github/workflows/molecule.yml' + - 'requirements.txt' + pull_request: + branches: [master] + paths: + - 'roles/vector/**' + - 'molecule/default/**' + - '.github/workflows/molecule.yml' + - 'requirements.txt' + schedule: + - cron: '20 10 * * tue' + + +jobs: + build: + runs-on: ubuntu-latest + env: + PY_COLORS: 1 + ANSIBLE_FORCE_COLOR: 1 + strategy: + fail-fast: false + matrix: + include: + - molecule_distro: rockylinux8 + molecule_docker_command: /usr/lib/systemd/systemd + - molecule_distro: rockylinux9 + molecule_docker_command: /usr/lib/systemd/systemd + molecule_distro: + - debian12 + - ubuntu2204 + - ubuntu2404 + molecule_docker_command: + - "/bin/systemd" + steps: + - name: Checkout repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + path: ansible_collections/telekom_mms/vector + submodules: true + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: 3.12 + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + working-directory: ansible_collections/telekom_mms/vector + + - name: Downgrade Ansible for Rocky 8 tests + run: pip install "ansible-core<2.17" + working-directory: ansible_collections/telekom_mms/vector + if: matrix.molecule_distro == 'rockylinux8' + + - name: Test with molecule + run: molecule test -s vector + env: + MOLECULE_DISTRO: ${{ matrix.molecule_distro }} + MOLECULE_DOCKER_COMMAND: ${{ matrix.molecule_docker_command }} + working-directory: ansible_collections/telekom_mms/vector diff --git a/.github/workflows/new_version.yml b/.github/workflows/new_version.yml deleted file mode 100644 index 5a0be23..0000000 --- a/.github/workflows/new_version.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -name: publish - -on: - push: - tags: - - v* - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - # Sets RELEASE_VERSION to be refs/tags/v -> - - name: Get the version from the tag - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV - - - name: Upload to ansible-galaxy - uses: artis3n/ansible_galaxy_collection@v2 - with: - api_key: '${{ secrets.GALAXY_API_KEY }}' - galaxy_version: '${{ env.RELEASE_VERSION }}' diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index 182258c..0000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Tests - -on: - push: - pull_request: - schedule: - - cron: '20 10 * * tue' -jobs: - build: - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip3 install -r requirements-tests.txt - ansible --version - ansible-lint --version - - name: Test - run: | - cd roles/vector && molecule test --all - - diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..821e42d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# 1.0.0 + +* initial release diff --git a/README.md b/README.md index 50344d3..20a2e7b 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,34 @@ -![publish](https://github.com/dzervas/ansible-vector/workflows/publish/badge.svg) +[![molecule](https://github.com/telekom-mms/ansible-vector/actions/workflows/molecule.yml/badge.svg)](https://github.com/telekom-mms/ansible-vector/actions/workflows/molecule.yml) [![ansible-lint](https://github.com/telekom-mms/ansible-vector/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/telekom-mms/ansible-vector/actions/workflows/ansible-lint.yml) -# Vector ansible role +# ansible-vector -This is an ansible role to set up [vector](https://vector.dev). -It translates the YAML configuration to TOML, so any configuration is possible. +This is a ansible collection to set up [vector](https://vector.dev) on various systems. -Currently only amd64, arch64, arch7 through deb and rpm packages are supported +## supported & tested OS + +| OS | Tested +|--------------|-------------------- +| Debian 12 | :white_check_mark: +| RL8 | :white_check_mark: +| RL9 | :white_check_mark: +| Ubuntu 22.04 | :white_check_mark: +| Ubuntu 24.04 | :white_check_mark: ## Variables -| Variable | Required | Default | Description -|--------------------------------------------|----------|------------------------|------------ -| vector_template | yes | vector.toml.j2 | path of your vector.toml template -| vector_config_file | yes | /etc/vector/vector.toml | system path of your vector.toml configuration -| vector_nightly | no | false | use vector nightly build -| add_vector_docker_group | no | false | add user vector to group docker -| add_vector_journal_group | no | false | add user vector to group systemd-journal -| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories -| vector_repo_key | no | `https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key` | configurable repo key, in case repo proxy is used -| vector_repo | no | Debian: `deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main`
Redhat: `https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch` | configurable repo, in case repo proxy is used -| vector_package | no | vector | option to define vector version with package name -| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) -| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) -| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) +| Variable | Required | Default | Description +|--------------------------|----------|-------------------------|------------ +| vector_template | yes | vector.yaml.j2 | path of your vector.yaml template +| vector_config_file | yes | /etc/vector/vector.yaml | system path of your vector.yaml configuration +| vector_groups | no | | add user vector to specified groups +| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories +| vector_repo_key | no | see `defaults/main.yml` | configurable repo key, in case repo proxy is used +| vector_repo | no | see `defaults/main.yml` | configurable repo, in case repo proxy is used +| vector_package | no | vector | option to define vector version with package name +| vector_version | no | | define vector version while vector is installed by source +| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) +| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) +| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) ## Example for configuration with ansible ```yaml @@ -47,7 +53,7 @@ sinks: ## Example playbook ```yaml -- name: install and configure vector +- name: Install and configure vector hosts: all vars: sources: @@ -66,5 +72,5 @@ sinks: inputs: ["journald"] address: "vector.example.com:9000" roles: - - vector + - telekom_mms.vector.vector ``` diff --git a/galaxy.yml b/galaxy.yml index 86bf2eb..c56e4f1 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,21 +1,24 @@ --- - -namespace: dzervas +namespace: telekom_mms name: vector +version: 1.0.0 readme: README.md authors: - Dimitris Zervas - Gaëtan Duchaussois - Dylan Humphreys - - Daniel Uhlmann - - Christopher Grau -description: vector.dev ansible role + - Daniel Uhlmann + - Christopher Grau +description: >- + This is an ansible collection to set up [vector](https://vector.dev) on various systems. license_file: LICENSE tags: - vector - logging - monitoring -repository: https://github.com/dzervas/ansible-vector -documentation: https://github.com/dzervas/ansible-vector -issues: https://github.com/dzervas/ansible-vector/issues -version: 0.1.6 +dependencies: + community.general: ">=8.6.0" +repository: https://github.com/telekom-mms/ansible-vector +documentation: https://github.com/telekom-mms/ansible-vector +homepage: https://github.com/telekom-mms/ansible-vector +issues: https://github.com/telekom-mms/ansible-vector/issues diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..765e8c9 --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + namespace: telekom_mms + role_name: vector + author: >- + Dimitris Zervas + Gaëtan Duchaussois + Dylan Humphreys + Daniel Uhlmann + Christopher Grau + description: >- + This is an ansible collection to set up [vector](https://vector.dev) on various systems. + license: LICENSE + min_ansible_version: "2.17" +dependencies: + - name: community.general + version: 8.6.0 diff --git a/meta/runtime.yml b/meta/runtime.yml index 938ca97..a764a08 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -1 +1,2 @@ -requires_ansible: '>=2.9.10' +--- +requires_ansible: ">=2.17.0" diff --git a/roles/vector/molecule/default/converge.yml b/molecule/vector/converge.yml similarity index 54% rename from roles/vector/molecule/default/converge.yml rename to molecule/vector/converge.yml index e59ddbb..4bed684 100644 --- a/roles/vector/molecule/default/converge.yml +++ b/molecule/vector/converge.yml @@ -6,18 +6,15 @@ journald: type: journald current_boot_only: true - transforms: - grok: - type: grok_parser - inputs: - - journald - pattern: '(?\\d+)%{GREEDYDATA}' sinks: vector: type: vector inputs: ["journald"] address: "vector.example.com:9000" + healthcheck: + enabled: false + vector_install_from_repo: true tasks: - - name: "Include vector" - include_role: - name: "vector" + - name: Include role vector + ansible.builtin.include_role: + name: telekom_mms.vector.vector diff --git a/molecule/vector/molecule.yml b/molecule/vector/molecule.yml new file mode 100644 index 0000000..0d106ab --- /dev/null +++ b/molecule/vector/molecule.yml @@ -0,0 +1,37 @@ +--- +driver: + name: docker + +platforms: + - name: instance + image: geerlingguy/docker-${MOLECULE_DISTRO}-ansible:latest + command: ${MOLECULE_DOCKER_COMMAND:-/bin/systemd} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + privileged: true + cgroupns_mode: host + pre_build_image: true + +provisioner: + name: ansible + config_options: + defaults: + deprecation_warnings: false + callback_whitelist: timer,profile_tasks + fact_caching: jsonfile + fact_caching_connection: ./cache + forks: 100 + connection: + pipelining: true +verifier: + name: ansible + +scenario: + test_sequence: + - dependency + - syntax + - create + - prepare + - converge + - verify + - destroy diff --git a/molecule/vector/verify.yml b/molecule/vector/verify.yml new file mode 100644 index 0000000..6943b14 --- /dev/null +++ b/molecule/vector/verify.yml @@ -0,0 +1,31 @@ +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Verify existing configuration file /etc/vector/vector.yaml + ansible.builtin.file: + path: /etc/vector/vector.yaml + state: file + register: result_vector_config_file + + - name: Validate existing configuration file /etc/vector/vector.yaml + ansible.builtin.assert: + that: + - result_vector_config_file.state == 'file' + - result_vector_config_file.owner == 'root' + - result_vector_config_file.group == 'root' + - result_vector_config_file.mode == '0644' + + - name: Verify that serivce vector is running + ansible.builtin.service: + name: vector + state: started + register: result_vector_service + + - name: Validate that serivce vector is running + ansible.builtin.assert: + that: + - result_vector_service.state == 'started' + - result_vector_service.changed == false + - result_vector_service.name == 'vector' diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index 5045816..0000000 --- a/requirements-tests.txt +++ /dev/null @@ -1,4 +0,0 @@ -ansible -ansible-lint -molecule[docker] -flake8 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..95eb492 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +molecule==24.12.0 +molecule-plugins[docker]==23.5.3 +ansible-core==2.18.1 +docker==7.1.0 diff --git a/roles/vector/.yamllint b/roles/vector/.yamllint deleted file mode 100644 index 8827676..0000000 --- a/roles/vector/.yamllint +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Based on ansible-lint config -extends: default - -rules: - braces: - max-spaces-inside: 1 - level: error - brackets: - max-spaces-inside: 1 - level: error - colons: - max-spaces-after: -1 - level: error - commas: - max-spaces-after: -1 - level: error - comments: disable - comments-indentation: disable - document-start: disable - empty-lines: - max: 3 - level: error - hyphens: - level: error - indentation: disable - key-duplicates: enable - line-length: disable - new-line-at-end-of-file: disable - new-lines: - type: unix - trailing-spaces: disable - truthy: disable diff --git a/roles/vector/README.md b/roles/vector/README.md deleted file mode 120000 index fe84005..0000000 --- a/roles/vector/README.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index d9f54aa..6aff384 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -1,14 +1,27 @@ -vector_template: vector.toml.j2 -vector_config_file: /etc/vector/vector.toml +--- -add_vector_docker_group: false -add_vector_journal_group: false +# vector configuration file +vector_template: vector.yaml.j2 +vector_config_file: /etc/vector/vector.yaml -vector_nightly: false -vector_version: "{{ vector_nightly | ternary('nightly','latest') }}" +# vector groups +# vector_groups: [] + +# install vector from repo vector_install_from_repo: false -vector_package: vector -vector_repo_key: "https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key" vector_repo: - Debian: "deb https://repositories.timber.io/public/vector/deb/{{ ansible_distribution | lower }} {{ ansible_lsb.codename | lower }} main" - RedHat: "https://repositories.timber.io/public/vector/rpm/el/$releasever/$basearch" + Debian: deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.vector.dev/ stable vector-0 + RedHat: https://yum.vector.dev/stable/vector-0/$basearch/ +vector_repo_key: + Debian: + - https://keys.datadoghq.com/DATADOG_APT_KEY_CURRENT.public + - https://keys.datadoghq.com/DATADOG_APT_KEY_F14F620E.public + - https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public + RedHat: + - https://keys.datadoghq.com/DATADOG_RPM_KEY_CURRENT.public + - https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public + - https://keys.datadoghq.com/DATADOG_RPM_KEY_FD4BF915.public +vector_repo_gpgcheck: true + +# vector package name +vector_package: vector diff --git a/roles/vector/handlers/main.yml b/roles/vector/handlers/main.yml index 4de2cc3..bd04dcb 100644 --- a/roles/vector/handlers/main.yml +++ b/roles/vector/handlers/main.yml @@ -1,5 +1,6 @@ -- name: restart vector - service: +--- +- name: Restart vector + ansible.builtin.service: state: restarted daemon_reload: true name: vector diff --git a/roles/vector/meta/main.yml b/roles/vector/meta/main.yml deleted file mode 100644 index cef098b..0000000 --- a/roles/vector/meta/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- - -galaxy_info: - role_name: vector - namespace: dzervas - author: Dimitris Zervas - description: vector.dev ansible role - license: MIT - platforms: - - name: Debian - versions: - - all - - name: Ubuntu - versions: - - all - galaxy_tags: - - vector - - logging - - monitoring - min_ansible_version: 2.9 diff --git a/roles/vector/molecule/default/molecule.yml b/roles/vector/molecule/default/molecule.yml deleted file mode 100644 index f8fa61a..0000000 --- a/roles/vector/molecule/default/molecule.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: | - set -e - yamllint . - ansible-lint - flake8 -platforms: - - name: debian10 - image: jrei/systemd-debian:10 - privileged: true - command: /lib/systemd/systemd - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: debian9 - image: jrei/systemd-debian:9 - privileged: true - command: /lib/systemd/systemd - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: ubuntu1804 - image: jrei/systemd-ubuntu:18.04 - privileged: true - command: /lib/systemd/systemd - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: ubuntu2004 - image: jrei/systemd-ubuntu:20.04 - privileged: true - command: /lib/systemd/systemd - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: rocky8 - image: rockylinux:8 - privileged: true - command: /usr/sbin/init - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - -provisioner: - name: ansible -verifier: - name: ansible diff --git a/roles/vector/molecule/default/verify.yml b/roles/vector/molecule/default/verify.yml deleted file mode 100644 index e5586a0..0000000 --- a/roles/vector/molecule/default/verify.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - - name: Execute vector - command: vector --version - changed_when: false - register: vector_version_rc - - name: Check grok pattern is not mangled - lineinfile: - path: /etc/vector/vector.toml - line: ' pattern = "(?\\d+)%{GREEDYDATA}"' - diff: true - register: grok_pattern_rc - - name: Assert vector is installed - assert: - that: - - vector_version_rc is success - - grok_pattern_rc is not changed diff --git a/roles/vector/molecule/features/converge.yml b/roles/vector/molecule/features/converge.yml deleted file mode 100644 index f1926d7..0000000 --- a/roles/vector/molecule/features/converge.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include vector" - include_role: - name: "vector" - vars: - vector_template: vector.yaml.j2 - vector_config_file: /etc/vector/vector.yaml - sources: - journald: - type: journald - current_boot_only: true - tcp: - type: socket - mode: tcp - address: 0.0.0.0:6000 - encoding: - codec: json - transforms: - grok: - type: grok_parser - inputs: - - journald - pattern: '(?\\d+)%{GREEDYDATA}' - sinks: - console: - type: console - encoding: - codec: json - inputs: - - grok diff --git a/roles/vector/molecule/features/molecule.yml b/roles/vector/molecule/features/molecule.yml deleted file mode 100644 index 14d5079..0000000 --- a/roles/vector/molecule/features/molecule.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: | - set -e - yamllint . - ansible-lint - flake8 -platforms: - - name: debian10 - image: jrei/systemd-debian:10 - privileged: true - command: /lib/systemd/systemd - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - name: rocky8 - image: rockylinux:8 - privileged: true - command: /usr/sbin/init - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro -provisioner: - name: ansible -verifier: - name: ansible diff --git a/roles/vector/molecule/features/verify.yml b/roles/vector/molecule/features/verify.yml deleted file mode 100644 index 8b8fffe..0000000 --- a/roles/vector/molecule/features/verify.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: true - tasks: - - name: Install netstat - yum: - name: - - net-tools - - procps - state: present - when: ansible_os_family == 'RedHat' - - name: Install netstat - apt: - name: - - net-tools - - procps - state: present - update_cache: yes - when: ansible_os_family == 'Debian' - - name: Execute vector - command: vector --version - changed_when: false - register: vector_version_rc - - name: Check port 6000 is open - community.general.listen_ports_facts: - - name: Assert vector is installed - assert: - that: - - vector_version_rc is success - - 6000 in open_ports - vars: - open_ports: "{{ ansible_facts.tcp_listen | map(attribute='port') }}" diff --git a/roles/vector/tasks/config.yml b/roles/vector/tasks/config.yml index 520e985..b1db6ae 100644 --- a/roles/vector/tasks/config.yml +++ b/roles/vector/tasks/config.yml @@ -1,29 +1,24 @@ --- -- name: Copy config - template: +- name: Create configuration + ansible.builtin.template: src: "{{ vector_template }}" dest: "{{ vector_config_file }}" - mode: 0644 - notify: restart vector + mode: "0644" + notify: Restart vector -- name: Add vector user to docker group - user: +- name: Add vector to specified groups + when: + - vector_groups is defined + - vector_groups | length > 0 + ansible.builtin.user: name: vector - groups: docker + groups: "{{ item }}" append: true - when: add_vector_docker_group | bool - notify: restart vector - -- name: Add vector user to systemd-journal group - user: - name: vector - groups: systemd-journal - append: true - when: add_vector_journal_group | bool - notify: restart vector + loop: "{{ vector_groups }}" + notify: Restart vector - name: Start vector - service: + ansible.builtin.service: state: started enabled: true name: vector diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml index 964f025..6f81a38 100644 --- a/roles/vector/tasks/install.yml +++ b/roles/vector/tasks/install.yml @@ -1,46 +1,31 @@ --- -- block: - # Workaround for latest version being named against a version number - - name: Get latest version - uri: - url: https://s3.amazonaws.com/packages.timber.io/?prefix=vector/latest&max-keys=1 - return_content: true - register: bucket_content - when: vector_version == "latest" - - name: Set latest version - set_fact: - vector_version: "{{ bucket_content.content | regex_replace('.*\\n.*[^-]+-(?P[\\d\\.]+)-.*','\\g') }}" - when: vector_version == "latest" - - - name: Install Vector (Debian) - apt: - deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb" +- name: Install vector from downloaded package + when: not vector_install_from_repo | bool + block: + - name: Install vector (Debian) + when: ansible_os_family == 'Debian' + ansible.builtin.apt: + deb: "https://apt.vector.dev/pool/v/ve/vector_{{ vector_version }}-1_{{ arch }}.deb" install_recommends: true - notify: - - restart vector + notify: Restart vector vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - arch: "{{ vector_debian_arch[ansible_machine] }}" - when: ansible_os_family == 'Debian' + arch: "{{ vector_arch[ansible_machine] }}" - - name: Install Vector (RedHat) - yum: - name: "https://packages.timber.io/vector/{{ version }}/vector-{{ package_version }}.{{ arch }}.rpm" + - name: Install vector (RedHat) + when: ansible_os_family == 'RedHat' + ansible.builtin.dnf: + name: "https://yum.vector.dev/stable/vector-0/{{ arch }}/vector-{{ vector_version }}-1.{{ arch }}.rpm" state: present disable_gpg_check: true # package is not signed - notify: - - restart vector + notify: Restart vector vars: - version: "{{ (vector_version == 'nightly')| bool | ternary('nightly/latest', vector_version) }}" - package_version: "{{ vector_version is match('latest') | ternary(vector_version, vector_version ~ '-1') }}" - arch: "{{ vector_redhat_arch[ansible_machine] }}" - when: ansible_os_family == 'RedHat' - when: vector_install_from_repo == false + arch: "{{ vector_arch[ansible_machine] }}" -- block: +- name: Install vector from repository + when: vector_install_from_repo | bool + block: - name: Install vector - package: + ansible.builtin.package: name: "{{ vector_package }}" state: present - when: vector_install_from_repo == true diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index 0f7ceb9..e515c01 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -1,11 +1,11 @@ --- -- name: Create repos for debian or redhat based systems - include_tasks: repo.yml +- name: Create repository for debian or redhat based systems when: vector_install_from_repo | bool + ansible.builtin.include_tasks: repo.yml - name: Install vector - include_tasks: install.yml + ansible.builtin.include_tasks: install.yml - name: Create configuration for vector - include_tasks: config.yml + ansible.builtin.include_tasks: config.yml diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index e35c05a..4fd5e95 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -1,25 +1,48 @@ --- -- block: - - name: Add vector gpg key to apt - become: true - apt_key: - url: "{{ vector_repo_key }}" +- name: Configure repository and Repository key on Debian + when: ansible_os_family == "Debian" + block: + - name: Ensure apt-transport-https curl and gnupg are installed + ansible.builtin.apt: + name: + - apt-transport-https + - curl + - gnupg state: present + update_cache: true + + - name: Create key file /usr/share/keyrings/datadog-archive-keyring.gpg + ansible.builtin.file: + path: /usr/share/keyrings/datadog-archive-keyring.gpg + mode: "0644" + owner: root + group: root + state: touch + + - name: Download key and put them into key file + ansible.builtin.shell: | + set -o pipefail + curl {{ item }} | gpg --no-default-keyring --keyring /usr/share/keyrings/datadog-archive-keyring.gpg --import --batch + register: download_repo_key + changed_when: true + failed_when: download_repo_key.rc != 0 + args: + executable: /bin/bash + loop: "{{ vector_repo_key[ansible_os_family] }}" - - name: Add vector repository for Debian based distributions - become: true - apt_repository: - repo: "{{ vector_repo }}" + - name: Add vector repository + ansible.builtin.apt_repository: + repo: "{{ vector_repo[ansible_os_family] }}" state: present - filename: timber-vector - when: ansible_os_family == "Debian" + filename: vector - name: Add vector repository for RedHat based distributions - yum_repository: - name: timber-vector - description: "Official repository for Timber / vector" - baseurl: "{{ vector_repo }}" - gpgcheck: true - gpgkey: "{{ vector_repo_key }}" when: ansible_os_family == "RedHat" + ansible.builtin.yum_repository: + name: vector + description: Vector + baseurl: "{{ vector_repo[ansible_os_family] }}" + gpgcheck: true + repo_gpgcheck: "{{ vector_repo_gpgcheck }}" + gpgkey: "{{ vector_repo_key[ansible_os_family] }}" diff --git a/roles/vector/vars/main.yml b/roles/vector/vars/main.yml deleted file mode 100644 index af90c4b..0000000 --- a/roles/vector/vars/main.yml +++ /dev/null @@ -1,8 +0,0 @@ -vector_debian_arch: - armv7l: armhf - aarch64: arm64 - x86_64: amd64 -vector_redhat_arch: - armv7l: armv7hl - aarch64: aarch64 - x86_64: x86_64 From 32be5175d85b917a63c93b8f7ec658e6623ae117 Mon Sep 17 00:00:00 2001 From: sjedlick <104558780+sjedlick@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:11:50 +0100 Subject: [PATCH 17/26] allow configuring RestartSec in systemd config (#5) * feat(NOISSUE): allow configuring RestartSec in systemd config --------- Co-authored-by: Swen Jedlicka Co-authored-by: schurzi --- .config/ansible-lint.yml | 1 + README.md | 27 +++++++++++---------- roles/vector/handlers/main.yml | 4 +++ roles/vector/tasks/config.yml | 20 +++++++++++++++ roles/vector/templates/vector_restartsec.j2 | 2 ++ 5 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 roles/vector/templates/vector_restartsec.j2 diff --git a/.config/ansible-lint.yml b/.config/ansible-lint.yml index b5937a0..75bec15 100644 --- a/.config/ansible-lint.yml +++ b/.config/ansible-lint.yml @@ -2,6 +2,7 @@ # .ansible-lint exclude_paths: - .cache/ # implicit unless exclude_paths is defined in config + - .ansible/ - molecule/ - .github/ diff --git a/README.md b/README.md index 20a2e7b..6877da7 100644 --- a/README.md +++ b/README.md @@ -16,19 +16,20 @@ This is a ansible collection to set up [vector](https://vector.dev) on various s ## Variables -| Variable | Required | Default | Description -|--------------------------|----------|-------------------------|------------ -| vector_template | yes | vector.yaml.j2 | path of your vector.yaml template -| vector_config_file | yes | /etc/vector/vector.yaml | system path of your vector.yaml configuration -| vector_groups | no | | add user vector to specified groups -| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories -| vector_repo_key | no | see `defaults/main.yml` | configurable repo key, in case repo proxy is used -| vector_repo | no | see `defaults/main.yml` | configurable repo, in case repo proxy is used -| vector_package | no | vector | option to define vector version with package name -| vector_version | no | | define vector version while vector is installed by source -| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) -| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) -| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) +| Variable | Required | Default | Description +|-------------------------------|----------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------- +| vector_template | yes | vector.yaml.j2 | path of your vector.yaml template +| vector_config_file | yes | /etc/vector/vector.yaml | system path of your vector.yaml configuration +| vector_groups | no | | add user vector to specified groups +| vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories +| vector_repo_key | no | see `defaults/main.yml` | configurable repo key, in case repo proxy is used +| vector_repo | no | see `defaults/main.yml` | configurable repo, in case repo proxy is used +| vector_restartsec | no | | add RestartSec to systemd config +| vector_package | no | vector | option to define vector version with package name +| vector_version | no | | define vector version while vector is installed by source +| sources | yes | false | ingest observability data from a wide variety of targets [link](https://vector.dev/docs/reference/configuration/sources/) +| transforms | no | false | shape your data as it moves through your Vector topology [link](https://vector.dev/docs/reference/configuration/transforms/) +| sinks | yes | false | deliver your observability data to a variety of destinations [link](https://vector.dev/docs/reference/configuration/sinks/) ## Example for configuration with ansible ```yaml diff --git a/roles/vector/handlers/main.yml b/roles/vector/handlers/main.yml index bd04dcb..0ab4aa7 100644 --- a/roles/vector/handlers/main.yml +++ b/roles/vector/handlers/main.yml @@ -1,4 +1,8 @@ --- +- name: Reload systemd + ansible.builtin.systemd_service: + daemon_reload: true + - name: Restart vector ansible.builtin.service: state: restarted diff --git a/roles/vector/tasks/config.yml b/roles/vector/tasks/config.yml index b1db6ae..a352b49 100644 --- a/roles/vector/tasks/config.yml +++ b/roles/vector/tasks/config.yml @@ -17,6 +17,26 @@ loop: "{{ vector_groups }}" notify: Restart vector +- name: Create a directory for drop-in configuration if it does not exist + when: + - vector_restartsec is defined + ansible.builtin.file: + path: /etc/systemd/system/vector.service.d + state: directory + owner: root + mode: '0755' + +- name: Add RestartSec to systemd config + when: + - vector_restartsec is defined + ansible.builtin.template: + src: vector_restartsec.j2 + dest: /etc/systemd/system/vector.service.d/vector.conf + owner: root + group: root + mode: '0644' + notify: Reload systemd + - name: Start vector ansible.builtin.service: state: started diff --git a/roles/vector/templates/vector_restartsec.j2 b/roles/vector/templates/vector_restartsec.j2 new file mode 100644 index 0000000..8706475 --- /dev/null +++ b/roles/vector/templates/vector_restartsec.j2 @@ -0,0 +1,2 @@ +[Service] +RestartSec={{ vector_restartsec }} From a40d4c062120c76efa2f66fb018beadc90f6f7ac Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 12 Nov 2025 11:36:40 +0100 Subject: [PATCH 18/26] separation of repo configuration and installation --- README.md | 1 + roles/vector/defaults/main.yml | 1 + roles/vector/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6877da7..9876269 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ This is a ansible collection to set up [vector](https://vector.dev) on various s | vector_template | yes | vector.yaml.j2 | path of your vector.yaml template | vector_config_file | yes | /etc/vector/vector.yaml | system path of your vector.yaml configuration | vector_groups | no | | add user vector to specified groups +| vector_manage_repo | no | false | configure deb or redhat based repositories | vector_install_from_repo | no | false | whether to install vector from packages or install from deb or redhat based repositories | vector_repo_key | no | see `defaults/main.yml` | configurable repo key, in case repo proxy is used | vector_repo | no | see `defaults/main.yml` | configurable repo, in case repo proxy is used diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml index 6aff384..b2ef4c3 100644 --- a/roles/vector/defaults/main.yml +++ b/roles/vector/defaults/main.yml @@ -9,6 +9,7 @@ vector_config_file: /etc/vector/vector.yaml # install vector from repo vector_install_from_repo: false +vector_manage_repo: false vector_repo: Debian: deb [signed-by=/usr/share/keyrings/datadog-archive-keyring.gpg] https://apt.vector.dev/ stable vector-0 RedHat: https://yum.vector.dev/stable/vector-0/$basearch/ diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml index e515c01..99a8c87 100644 --- a/roles/vector/tasks/main.yml +++ b/roles/vector/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Create repository for debian or redhat based systems - when: vector_install_from_repo | bool + when: vector_manage_repo | bool ansible.builtin.include_tasks: repo.yml - name: Install vector From e1efac7c55c1c7ee660ac671152092e4e6031a25 Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 12 Nov 2025 12:43:57 +0100 Subject: [PATCH 19/26] fix ansible-lint action --- .github/workflows/ansible-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index 71cdfe7..57dfbdf 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -29,4 +29,4 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Lint Ansible Playbook - uses: ansible/ansible-lint@44be233dbd6a8a6d8f3c5297c318ed4ed4644c32 # v24 + uses: ansible/ansible-lint@v25.11.0 # 25 From 1ab693cbbea19d8d99d7ce992fb2dbe22ce6d7d8 Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 12 Nov 2025 12:45:40 +0100 Subject: [PATCH 20/26] fix var-naming[no-role-prefix]: Variables names from within roles should use vector_ as a prefix. (register: download_repo_key) --- roles/vector/tasks/repo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index 4fd5e95..d633b3d 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -24,9 +24,9 @@ ansible.builtin.shell: | set -o pipefail curl {{ item }} | gpg --no-default-keyring --keyring /usr/share/keyrings/datadog-archive-keyring.gpg --import --batch - register: download_repo_key + register: vector_download_repo_key changed_when: true - failed_when: download_repo_key.rc != 0 + failed_when: vector_download_repo_key.rc != 0 args: executable: /bin/bash loop: "{{ vector_repo_key[ansible_os_family] }}" From 25b0a1f5a44e5aafdc3d01f1e9c98c9960eb3322 Mon Sep 17 00:00:00 2001 From: beechesII Date: Wed, 12 Nov 2025 12:49:59 +0100 Subject: [PATCH 21/26] bump version to 2.0.0 --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index c56e4f1..e5f02fd 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,7 +1,7 @@ --- namespace: telekom_mms name: vector -version: 1.0.0 +version: 2.0.0 readme: README.md authors: - Dimitris Zervas From 3d415f0bb2df9fdc35210e833283b6e178614a23 Mon Sep 17 00:00:00 2001 From: beechesII Date: Thu, 13 Nov 2025 20:38:27 +0100 Subject: [PATCH 22/26] add vector_manage_repo to molecule test case --- molecule/vector/converge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/molecule/vector/converge.yml b/molecule/vector/converge.yml index 4bed684..1dd4cd3 100644 --- a/molecule/vector/converge.yml +++ b/molecule/vector/converge.yml @@ -14,6 +14,7 @@ healthcheck: enabled: false vector_install_from_repo: true + vector_manage_repo: true tasks: - name: Include role vector ansible.builtin.include_role: From 55c0518b40637a5301b8d72b4709554d23f9be9a Mon Sep 17 00:00:00 2001 From: beechesII Date: Thu, 13 Nov 2025 20:54:41 +0100 Subject: [PATCH 23/26] enhancement - add molecule test for debian 13 --- .github/workflows/molecule.yml | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index a8cfe89..20df2d8 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -36,6 +36,7 @@ jobs: molecule_docker_command: /usr/lib/systemd/systemd molecule_distro: - debian12 + - debian13 - ubuntu2204 - ubuntu2404 molecule_docker_command: diff --git a/README.md b/README.md index 9876269..5551aed 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This is a ansible collection to set up [vector](https://vector.dev) on various s | OS | Tested |--------------|-------------------- | Debian 12 | :white_check_mark: +| Debian 13 | :white_check_mark: | RL8 | :white_check_mark: | RL9 | :white_check_mark: | Ubuntu 22.04 | :white_check_mark: From fdb449d80d4af20570793b010731d165b40d2c65 Mon Sep 17 00:00:00 2001 From: beechesII Date: Thu, 13 Nov 2025 21:16:05 +0100 Subject: [PATCH 24/26] fix instance creation --- .github/workflows/molecule.yml | 3 ++- galaxy.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 20df2d8..85d7895 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -34,9 +34,10 @@ jobs: molecule_docker_command: /usr/lib/systemd/systemd - molecule_distro: rockylinux9 molecule_docker_command: /usr/lib/systemd/systemd + - molecule_distro: debian13 + molecule_docker_command: /usr/lib/systemd/systemd molecule_distro: - debian12 - - debian13 - ubuntu2204 - ubuntu2404 molecule_docker_command: diff --git a/galaxy.yml b/galaxy.yml index e5f02fd..cb5fa65 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,7 +1,7 @@ --- namespace: telekom_mms name: vector -version: 2.0.0 +version: 2.0.1 readme: README.md authors: - Dimitris Zervas From 1c636ad12c79f3f6bd0e8ca3282ecc7091c56ebf Mon Sep 17 00:00:00 2001 From: Swen Jedlicka Date: Tue, 20 Jan 2026 15:51:06 +0100 Subject: [PATCH 25/26] fix: update syntax for ansible_facts in install and repo tasks --- roles/vector/tasks/install.yml | 8 ++++---- roles/vector/tasks/repo.yml | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml index 6f81a38..071d83e 100644 --- a/roles/vector/tasks/install.yml +++ b/roles/vector/tasks/install.yml @@ -4,23 +4,23 @@ when: not vector_install_from_repo | bool block: - name: Install vector (Debian) - when: ansible_os_family == 'Debian' + when: ansible_facts.os_family == 'Debian' ansible.builtin.apt: deb: "https://apt.vector.dev/pool/v/ve/vector_{{ vector_version }}-1_{{ arch }}.deb" install_recommends: true notify: Restart vector vars: - arch: "{{ vector_arch[ansible_machine] }}" + arch: "{{ vector_arch[ansible_facts.machine] }}" - name: Install vector (RedHat) - when: ansible_os_family == 'RedHat' + when: ansible_facts.os_family == 'RedHat' ansible.builtin.dnf: name: "https://yum.vector.dev/stable/vector-0/{{ arch }}/vector-{{ vector_version }}-1.{{ arch }}.rpm" state: present disable_gpg_check: true # package is not signed notify: Restart vector vars: - arch: "{{ vector_arch[ansible_machine] }}" + arch: "{{ vector_arch[ansible_facts.machine] }}" - name: Install vector from repository when: vector_install_from_repo | bool diff --git a/roles/vector/tasks/repo.yml b/roles/vector/tasks/repo.yml index d633b3d..ff5d79f 100644 --- a/roles/vector/tasks/repo.yml +++ b/roles/vector/tasks/repo.yml @@ -1,7 +1,7 @@ --- - name: Configure repository and Repository key on Debian - when: ansible_os_family == "Debian" + when: ansible_facts.os_family == "Debian" block: - name: Ensure apt-transport-https curl and gnupg are installed ansible.builtin.apt: @@ -29,20 +29,20 @@ failed_when: vector_download_repo_key.rc != 0 args: executable: /bin/bash - loop: "{{ vector_repo_key[ansible_os_family] }}" + loop: "{{ vector_repo_key[ansible_facts.os_family] }}" - name: Add vector repository ansible.builtin.apt_repository: - repo: "{{ vector_repo[ansible_os_family] }}" + repo: "{{ vector_repo[ansible_facts.os_family] }}" state: present filename: vector - name: Add vector repository for RedHat based distributions - when: ansible_os_family == "RedHat" + when: ansible_facts.os_family == "RedHat" ansible.builtin.yum_repository: name: vector description: Vector - baseurl: "{{ vector_repo[ansible_os_family] }}" + baseurl: "{{ vector_repo[ansible_facts.os_family] }}" gpgcheck: true repo_gpgcheck: "{{ vector_repo_gpgcheck }}" - gpgkey: "{{ vector_repo_key[ansible_os_family] }}" + gpgkey: "{{ vector_repo_key[ansible_facts.os_family] }}" From e6ec9ba864d2ea7d22e8ffb1a261cc645b8d17ab Mon Sep 17 00:00:00 2001 From: Christopher Grau <66788631+beechesII@users.noreply.github.com> Date: Thu, 19 Mar 2026 16:29:23 +0100 Subject: [PATCH 26/26] fix - meta/main.yml (#10) Co-authored-by: Christopher Grau --- galaxy.yml | 2 +- meta/main.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index cb5fa65..989beb9 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,7 +1,7 @@ --- namespace: telekom_mms name: vector -version: 2.0.1 +version: 2.0.2 readme: README.md authors: - Dimitris Zervas diff --git a/meta/main.yml b/meta/main.yml index 765e8c9..ac77637 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -12,6 +12,3 @@ galaxy_info: This is an ansible collection to set up [vector](https://vector.dev) on various systems. license: LICENSE min_ansible_version: "2.17" -dependencies: - - name: community.general - version: 8.6.0