diff --git a/cfg/config.go b/cfg/config.go index 0860ac885..fa0279c3a 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -23,7 +23,7 @@ import ( const trueValue = "true" type Config struct { - Vault VaultConfig + Vault *VaultConfig Pipelines map[string]*PipelineConfig } @@ -65,24 +65,23 @@ type PipelineConfig struct { } type VaultConfig struct { - Token string - Address string - ShouldUse bool + Address string + + Token string + + AuthMountPath string + RoleID string + SecretID string } func NewConfig() *Config { return &Config{ - Vault: VaultConfig{ - Token: "", - Address: "", - ShouldUse: false, - }, Pipelines: make(map[string]*PipelineConfig, 20), } } -func NewConfigFromFile(paths []string) *Config { - mergedConfig := make(map[interface{}]interface{}) +func NewConfigFromFiles(paths []string) *Config { + mergedConfig := make(map[any]any, len(paths)) for _, path := range paths { logger.Infof("reading config %q", path) @@ -90,7 +89,7 @@ func NewConfigFromFile(paths []string) *Config { if err != nil { logger.Fatalf("can't read config file %q: %s", path, err) } - var currentConfig map[interface{}]interface{} + var currentConfig map[any]any if err := yaml.Unmarshal(yamlContents, ¤tConfig); err != nil { logger.Fatalf("can't parse config file yaml %q: %s", path, err) } @@ -123,8 +122,8 @@ func NewConfigFromFile(paths []string) *Config { // if vault is used then set value otherwise it is empty variable vault := &vault{} - if config.Vault.ShouldUse { - vault, err = newVault(config.Vault.Address, config.Vault.Token) + if config.Vault != nil { + vault, err = newVault(config.Vault) if err != nil { logger.Fatalf("can't create vault client: %s", err.Error()) } @@ -164,26 +163,14 @@ func applyEnvs(object *simplejson.Json) error { } func parseConfig(object *simplejson.Json) *Config { - config := NewConfig() - vault := object.Get("vault") var err error - addr := vault.Get("address") - if addr.Interface() != nil { - config.Vault.Address, err = addr.String() - if err != nil { - logger.Panicf("can't parse vault address: %s", err.Error()) - } - } + config := NewConfig() - token := vault.Get("token") - if token.Interface() != nil { - config.Vault.Token, err = token.String() - if err != nil { - logger.Panicf("can't parse vault token: %s", err.Error()) - } + config.Vault, err = parseVaultConfig(object.Get("vault")) + if err != nil { + logger.Fatalf("can't parse vault config: %s", err.Error()) } - config.Vault.ShouldUse = config.Vault.Address != "" && config.Vault.Token != "" pipelinesJson := object.Get("pipelines") pipelines := pipelinesJson.MustMap() @@ -201,6 +188,35 @@ func parseConfig(object *simplejson.Json) *Config { return config } +func parseVaultConfig(vault *simplejson.Json) (*VaultConfig, error) { + if vault.Interface() == nil { + return nil, nil + } + + vc := &VaultConfig{ + Address: vault.Get("address").MustString(), + + Token: vault.Get("token").MustString(), + + AuthMountPath: vault.Get("auth_mount_path").MustString(), + RoleID: vault.Get("role_id").MustString(), + SecretID: vault.Get("secret_id").MustString(), + } + + if vc.Address == "" { + return nil, errors.New("vault address must be non-empty string") + } + if vc.Token != "" { + return vc, nil + } + + if vc.AuthMountPath == "" && vc.RoleID == "" && vc.SecretID == "" { + return nil, errors.New("one of auth methods must be specified: token or role&secret") + } + + return vc, nil +} + func validatePipelineName(name string) error { matched, err := regexp.MatchString("^[a-zA-Z0-9_]+$", name) if err != nil { diff --git a/cfg/config_test.go b/cfg/config_test.go index 2031e66e5..37a725739 100644 --- a/cfg/config_test.go +++ b/cfg/config_test.go @@ -17,7 +17,7 @@ func NewTestConfig(names []string) *Config { for _, name := range names { configFiles = append(configFiles, "../testdata/config/"+name) } - return NewConfigFromFile(configFiles) + return NewConfigFromFiles(configFiles) } func TestSimple(t *testing.T) { diff --git a/cfg/vault.go b/cfg/vault.go index a7c5bac78..624d8fcaf 100644 --- a/cfg/vault.go +++ b/cfg/vault.go @@ -1,10 +1,12 @@ package cfg import ( + "context" "fmt" "strings" "github.com/hashicorp/vault/api" + auth "github.com/hashicorp/vault/api/auth/approle" "github.com/ozontech/file.d/logger" ) @@ -16,32 +18,53 @@ type vault struct { c *api.Client } -func newVault(addr, token string) (*vault, error) { +func newVault(cfg *VaultConfig) (*vault, error) { + if cfg == nil { + return nil, nil + } + conf := api.DefaultConfig() - conf.Address = addr + conf.Address = cfg.Address c, err := api.NewClient(conf) if err != nil { - return nil, fmt.Errorf("can't create client: %w", err) + return nil, fmt.Errorf("can't create api client: %w", err) } - c.SetToken(token) + if cfg.Token != "" { + c.SetToken(cfg.Token) + } else { + appRoleAuth, err := auth.NewAppRoleAuth( + cfg.RoleID, + &auth.SecretID{FromString: cfg.SecretID}, + auth.WithMountPath(cfg.AuthMountPath), + ) + if err != nil { + return nil, fmt.Errorf("can't create approle auth: %w", err) + } + + // after a successful login, this method will automatically set the client’s token + _, err = c.Auth().Login(context.Background(), appRoleAuth) + if err != nil { + return nil, err + } + } return &vault{c: c}, nil } func (v *vault) GetSecret(path, key string) (string, error) { - if v.c == nil { - logger.Fatalf("can't get secret without connection vault api.Client") + if v == nil || v.c == nil { + logger.Fatalf("can't get secret without vault api client") } - c := v.c - secret, err := c.Logical().Read(path) + + secret, err := v.c.Logical().Read(path) if err != nil { - return "", fmt.Errorf("can't get secret: %w", err) + return "", fmt.Errorf("can't get secret %q: %w", path, err) } str, ok := secret.Data[key].(string) if !ok { - return "", fmt.Errorf("can't get 'key' of the secret: %q", key) + return "", fmt.Errorf("can't get key %q of the secret %q", key, path) } return str, nil diff --git a/cmd/file.d/file.d.go b/cmd/file.d/file.d.go index 4cb8b58ca..65c269e58 100644 --- a/cmd/file.d/file.d.go +++ b/cmd/file.d/file.d.go @@ -111,7 +111,7 @@ func main() { } func start() { - appCfg := cfg.NewConfigFromFile(*config) + appCfg := cfg.NewConfigFromFiles(*config) fileD = fd.New(appCfg, *http) fileD.Start() diff --git a/cmd/file.d/file.d_test.go b/cmd/file.d/file.d_test.go index 3235eac24..749c6e82f 100644 --- a/cmd/file.d/file.d_test.go +++ b/cmd/file.d/file.d_test.go @@ -86,7 +86,7 @@ func TestEndToEnd(t *testing.T) { filesDir := t.TempDir() offsetsDir := t.TempDir() - config := cfg.NewConfigFromFile([]string{configFilename, configOverrideFilename}) + config := cfg.NewConfigFromFiles([]string{configFilename, configOverrideFilename}) input := config.Pipelines["test"].Raw.Get("input") input.Set("watching_dir", filesDir) input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml")) diff --git a/docs/configuring.idoc.md b/docs/configuring.idoc.md index da0973c88..21910691f 100644 --- a/docs/configuring.idoc.md +++ b/docs/configuring.idoc.md @@ -1,18 +1,15 @@ # Configuring - You can specify several pipelines with plugins and their parameters in a yaml format. Examples can be found [here](/docs/examples.md). -### Logging - +## Logging Logging is configured with `LOG_LEVEL` environment variable ('info' by default). Logging level can be changed in runtime with [standard zap handler](https://github.com/uber-go/zap/blob/v1.23.0/http_handler.go#L33-L70) exposed at `/log/level`. -### Actions debugging - +## Actions debugging To debug any working action you must enable http server via '-http' flag and visit an endpoint `/pipelines///sample`. It will show 1 sample of the in/out events. @@ -46,8 +43,7 @@ If `-http=':9090'` debug endpoints will be: > **Note**: by default debug server starts on address `:9000` (can be checked using `--help` flag). If your server uses IPv6, the debug server with address configured as `:` will start on IPv6 port. -### Overriding configurations - +## Overriding configurations You can use multiple configuration files. This allows you to define a base configuration (e.g., common.yaml) and override or extend it with additional configurations (e.g., local.yaml). ``` @@ -88,8 +84,7 @@ pipelines: Arrays (or lists) are usually replaced entirely when merging configurations (e.g., actions). Dictionaries (or maps), on the other hand, are typically merged (e.g., output.type). -### Overriding by environment variables - +## Overriding by environment variables `file.d` can override config fields if you specify environment variables with `FILED_` prefix. The name of the env will be divided by underscores and the config will set or override the config field by the resulted path. @@ -107,14 +102,12 @@ pipelines: pipeline_name: ... ``` -### Vault support - +## Vault support Consider this config: - ```yaml vault: - token: example_token address: http://127.0.0.1:8200 + token: example_token pipelines: example: input: @@ -124,14 +117,27 @@ pipelines: type: devnull ``` -`file.d` supports getting secrets from Vault as soon as you specify Vault token and an address in a configuration. -Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`, -and `file.d` tries to connect to Vault and get the secret from there. -If you need to pass a literal string that begins with `vault(`, you should escape the value with a -backslash: `\vault(path/to/secret, key)`. +`file.d` supports getting secrets from Vault as soon as you specify Vault address and one of auth methods in a configuration. +Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`, and `file.d` tries to connect to Vault and get the secret from there. -### Env support +> If you need to pass a literal string that begins with `vault(`, you should escape the value with a backslash: `\vault(path/to/secret, key)`. +### Auth methods +Token: +```yaml +vault: + token: example_token +``` + +AppRole: +```yaml +vault: + role_id: example_role + secret_id: example_secret + auth_mount_path: some/path # used when formatting the authorization uri: 'auth/%s/login' +``` + +## Env support Consider this config: ```yaml @@ -146,16 +152,14 @@ pipelines: field: env(ENV_NAME) ``` -`file.d` supports getting environment variables. Then you can write any -field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`, -and `file.d` tries to get environment variable value. If you need to pass -a literal string that begins with `env(`, you should escape the value with a +`file.d` supports getting environment variables. Then you can write any field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`, and `file.d` tries to get environment variable value. + +> If you need to pass a literal string that begins with `env(`, you should escape the value with a backslash: `\env(ENV_NAME)`. -### Do action if match +## Do action if match ### match_fields - File.d can do any action if it matches by pattern. In the `match_fields` you can pass some patterns, for example: @@ -225,8 +229,7 @@ Patterns must have a list ([]) or string type, not a number or null. ### Match modes @match-modes|header-description -### Decoders - +## Decoders If you have logs in specific non-json format, you can specify decoder type in pipeline settings. By default `json` decoder is used. More details can be found [here](/decoder/readme.md). ```yml diff --git a/docs/configuring.md b/docs/configuring.md index c1f103f32..e0928a7ef 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -1,18 +1,15 @@ # Configuring - You can specify several pipelines with plugins and their parameters in a yaml format. Examples can be found [here](/docs/examples.md). -### Logging - +## Logging Logging is configured with `LOG_LEVEL` environment variable ('info' by default). Logging level can be changed in runtime with [standard zap handler](https://github.com/uber-go/zap/blob/v1.23.0/http_handler.go#L33-L70) exposed at `/log/level`. -### Actions debugging - +## Actions debugging To debug any working action you must enable http server via '-http' flag and visit an endpoint `/pipelines///sample`. It will show 1 sample of the in/out events. @@ -46,8 +43,7 @@ If `-http=':9090'` debug endpoints will be: > **Note**: by default debug server starts on address `:9000` (can be checked using `--help` flag). If your server uses IPv6, the debug server with address configured as `:` will start on IPv6 port. -### Overriding configurations - +## Overriding configurations You can use multiple configuration files. This allows you to define a base configuration (e.g., common.yaml) and override or extend it with additional configurations (e.g., local.yaml). ``` @@ -88,8 +84,7 @@ pipelines: Arrays (or lists) are usually replaced entirely when merging configurations (e.g., actions). Dictionaries (or maps), on the other hand, are typically merged (e.g., output.type). -### Overriding by environment variables - +## Overriding by environment variables `file.d` can override config fields if you specify environment variables with `FILED_` prefix. The name of the env will be divided by underscores and the config will set or override the config field by the resulted path. @@ -107,14 +102,12 @@ pipelines: pipeline_name: ... ``` -### Vault support - +## Vault support Consider this config: - ```yaml vault: - token: example_token address: http://127.0.0.1:8200 + token: example_token pipelines: example: input: @@ -124,14 +117,27 @@ pipelines: type: devnull ``` -`file.d` supports getting secrets from Vault as soon as you specify Vault token and an address in a configuration. -Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`, -and `file.d` tries to connect to Vault and get the secret from there. -If you need to pass a literal string that begins with `vault(`, you should escape the value with a -backslash: `\vault(path/to/secret, key)`. +`file.d` supports getting secrets from Vault as soon as you specify Vault address and one of auth methods in a configuration. +Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`, and `file.d` tries to connect to Vault and get the secret from there. -### Env support +> If you need to pass a literal string that begins with `vault(`, you should escape the value with a backslash: `\vault(path/to/secret, key)`. +### Auth methods +Token: +```yaml +vault: + token: example_token +``` + +AppRole: +```yaml +vault: + role_id: example_role + secret_id: example_secret + auth_mount_path: some/path # used when formatting the authorization uri: 'auth/%s/login' +``` + +## Env support Consider this config: ```yaml @@ -146,16 +152,14 @@ pipelines: field: env(ENV_NAME) ``` -`file.d` supports getting environment variables. Then you can write any -field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`, -and `file.d` tries to get environment variable value. If you need to pass -a literal string that begins with `env(`, you should escape the value with a +`file.d` supports getting environment variables. Then you can write any field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`, and `file.d` tries to get environment variable value. + +> If you need to pass a literal string that begins with `env(`, you should escape the value with a backslash: `\env(ENV_NAME)`. -### Do action if match +## Do action if match ### match_fields - File.d can do any action if it matches by pattern. In the `match_fields` you can pass some patterns, for example: @@ -329,8 +333,7 @@ result:
-### Decoders - +## Decoders If you have logs in specific non-json format, you can specify decoder type in pipeline settings. By default `json` decoder is used. More details can be found [here](/decoder/readme.md). ```yml diff --git a/e2e/start_work_test.go b/e2e/start_work_test.go index c7c349eb2..98cb8fe36 100644 --- a/e2e/start_work_test.go +++ b/e2e/start_work_test.go @@ -214,7 +214,7 @@ func TestE2EStabilityWorkCase(t *testing.T) { test := test num := num t.Run(test.name, func(t *testing.T) { - conf := cfg.NewConfigFromFile([]string{test.cfgPath}) + conf := cfg.NewConfigFromFiles([]string{test.cfgPath}) if _, ok := conf.Pipelines[test.name]; !ok { log.Fatalf("pipeline name must be named the same as the name of the test") } diff --git a/go.mod b/go.mod index 26f633e58..67162cced 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/vault/api v1.22.0 + github.com/hashicorp/vault/api/auth/approle v0.12.0 github.com/jackc/pgconn v1.14.3 github.com/jackc/pgproto3/v2 v2.3.3 github.com/jackc/pgx/v4 v4.18.3 diff --git a/go.sum b/go.sum index 5a8f56710..7b4964fef 100644 --- a/go.sum +++ b/go.sum @@ -149,6 +149,8 @@ github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0= github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM= +github.com/hashicorp/vault/api/auth/approle v0.12.0 h1:PhF7jrQjydK1DC05EboosXmZg31GDUIKL8bjyilsJ+E= +github.com/hashicorp/vault/api/auth/approle v0.12.0/go.mod h1:J7BJLpXeQXhuMAWi31Puunu5QOeCoRAgLh2iDti7OLA= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=