add vault approle auth method

cfg/config.go

@@ -23,7 +23,7 @@ import (
 const trueValue = "true"
 
 type Config struct {
-	Vault     VaultConfig
+	Vault     *VaultConfig
 	Pipelines map[string]*PipelineConfig
 }
 
@@ -65,32 +65,31 @@ type PipelineConfig struct {
 }
 
 type VaultConfig struct {
-	Token     string
-	Address   string
-	ShouldUse bool
+	Address string
+
+	Token string
+
+	AuthMountPath string
+	RoleID        string
+	SecretID      string
 }
 
 func NewConfig() *Config {
 	return &Config{
-		Vault: VaultConfig{
-			Token:     "",
-			Address:   "",
-			ShouldUse: false,
-		},
 		Pipelines: make(map[string]*PipelineConfig, 20),
 	}
 }
 
-func NewConfigFromFile(paths []string) *Config {
-	mergedConfig := make(map[interface{}]interface{})
+func NewConfigFromFiles(paths []string) *Config {
+	mergedConfig := make(map[any]any, len(paths))
 
 	for _, path := range paths {
 		logger.Infof("reading config %q", path)
 		yamlContents, err := os.ReadFile(path)
 		if err != nil {
 			logger.Fatalf("can't read config file %q: %s", path, err)
 		}
-		var currentConfig map[interface{}]interface{}
+		var currentConfig map[any]any
 		if err := yaml.Unmarshal(yamlContents, &currentConfig); err != nil {
 			logger.Fatalf("can't parse config file yaml %q: %s", path, err)
 		}
@@ -123,8 +122,8 @@ func NewConfigFromFile(paths []string) *Config {
 
 	// if vault is used then set value otherwise it is empty variable
 	vault := &vault{}
-	if config.Vault.ShouldUse {
-		vault, err = newVault(config.Vault.Address, config.Vault.Token)
+	if config.Vault != nil {
+		vault, err = newVault(config.Vault)
 		if err != nil {
 			logger.Fatalf("can't create vault client: %s", err.Error())
 		}
@@ -164,26 +163,14 @@ func applyEnvs(object *simplejson.Json) error {
 }
 
 func parseConfig(object *simplejson.Json) *Config {
-	config := NewConfig()
-	vault := object.Get("vault")
 	var err error
 
-	addr := vault.Get("address")
-	if addr.Interface() != nil {
-		config.Vault.Address, err = addr.String()
-		if err != nil {
-			logger.Panicf("can't parse vault address: %s", err.Error())
-		}
-	}
+	config := NewConfig()
 
-	token := vault.Get("token")
-	if token.Interface() != nil {
-		config.Vault.Token, err = token.String()
-		if err != nil {
-			logger.Panicf("can't parse vault token: %s", err.Error())
-		}
+	config.Vault, err = parseVaultConfig(object.Get("vault"))
+	if err != nil {
+		logger.Panicf("can't parse vault config: %s", err.Error())
 	}
-	config.Vault.ShouldUse = config.Vault.Address != "" && config.Vault.Token != ""
 
 	pipelinesJson := object.Get("pipelines")
 	pipelines := pipelinesJson.MustMap()
@@ -201,6 +188,35 @@ func parseConfig(object *simplejson.Json) *Config {
 	return config
 }
 
+func parseVaultConfig(vault *simplejson.Json) (*VaultConfig, error) {
+	if vault.Interface() == nil {
+		return nil, nil
+	}
+
+	vc := &VaultConfig{
+		Address: vault.Get("address").MustString(),
+
+		Token: vault.Get("token").MustString(),
+
+		AuthMountPath: vault.Get("auth_mount_path").MustString(),
+		RoleID:        vault.Get("role_id").MustString(),
+		SecretID:      vault.Get("secret_id").MustString(),
+	}
+
+	if vc.Address == "" {
+		return nil, errors.New("vault address must be non-empty string")
+	}
+	if vc.Token != "" {
+		return vc, nil
+	}
+
+	if vc.AuthMountPath == "" && vc.RoleID == "" && vc.SecretID == "" {
+		return nil, errors.New("one of auth methods must be specified: token or role&secret")
+	}
+
+	return vc, nil
+}
+
 func validatePipelineName(name string) error {
 	matched, err := regexp.MatchString("^[a-zA-Z0-9_]+$", name)
 	if err != nil {

cfg/config_test.go

@@ -17,7 +17,7 @@ func NewTestConfig(names []string) *Config {
 	for _, name := range names {
 		configFiles = append(configFiles, "../testdata/config/"+name)
 	}
-	return NewConfigFromFile(configFiles)
+	return NewConfigFromFiles(configFiles)
 }
 
 func TestSimple(t *testing.T) {

cfg/vault.go

@@ -1,10 +1,12 @@
 package cfg
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	"github.com/hashicorp/vault/api"
+	auth "github.com/hashicorp/vault/api/auth/approle"
 	"github.com/ozontech/file.d/logger"
 )
 
@@ -16,32 +18,53 @@ type vault struct {
 	c *api.Client
 }
 
-func newVault(addr, token string) (*vault, error) {
+func newVault(cfg *VaultConfig) (*vault, error) {
+	if cfg == nil {
+		return nil, nil
+	}
+
 	conf := api.DefaultConfig()
-	conf.Address = addr
+	conf.Address = cfg.Address
 	c, err := api.NewClient(conf)
 	if err != nil {
-		return nil, fmt.Errorf("can't create client: %w", err)
+		return nil, fmt.Errorf("can't create api client: %w", err)
 	}
 
-	c.SetToken(token)
+	if cfg.Token != "" {
+		c.SetToken(cfg.Token)
+	} else {
+		appRoleAuth, err := auth.NewAppRoleAuth(
+			cfg.RoleID,
+			&auth.SecretID{FromString: cfg.SecretID},
+			auth.WithMountPath(cfg.AuthMountPath),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("can't create approle auth: %w", err)
+		}
+
+		// after a successful login, this method will automatically set the client’s token
+		_, err = c.Auth().Login(context.Background(), appRoleAuth)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	return &vault{c: c}, nil
 }
 
 func (v *vault) GetSecret(path, key string) (string, error) {
-	if v.c == nil {
-		logger.Fatalf("can't get secret without connection vault api.Client")
+	if v == nil || v.c == nil {
+		logger.Fatalf("can't get secret without vault api client")
 	}
-	c := v.c
-	secret, err := c.Logical().Read(path)
+
+	secret, err := v.c.Logical().Read(path)
 	if err != nil {
-		return "", fmt.Errorf("can't get secret: %w", err)
+		return "", fmt.Errorf("can't get secret %q: %w", path, err)
 	}
 
 	str, ok := secret.Data[key].(string)
 	if !ok {
-		return "", fmt.Errorf("can't get 'key' of the secret: %q", key)
+		return "", fmt.Errorf("can't get key %q of the secret %q", key, path)
 	}
 
 	return str, nil

cmd/file.d/file.d.go

@@ -111,7 +111,7 @@ func main() {
 }
 
 func start() {
-	appCfg := cfg.NewConfigFromFile(*config)
+	appCfg := cfg.NewConfigFromFiles(*config)
 
 	fileD = fd.New(appCfg, *http)
 	fileD.Start()

cmd/file.d/file.d_test.go

@@ -86,7 +86,7 @@ func TestEndToEnd(t *testing.T) {
 	filesDir := t.TempDir()
 	offsetsDir := t.TempDir()
 
-	config := cfg.NewConfigFromFile([]string{configFilename, configOverrideFilename})
+	config := cfg.NewConfigFromFiles([]string{configFilename, configOverrideFilename})
 	input := config.Pipelines["test"].Raw.Get("input")
 	input.Set("watching_dir", filesDir)
 	input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml"))

docs/configuring.idoc.md

@@ -1,18 +1,15 @@
 # Configuring
-
 You can specify several pipelines with plugins and their parameters in a yaml format.  
 Examples can be found [here](/docs/examples.md).
 
-### Logging
-
+## Logging
 Logging is configured with `LOG_LEVEL` environment variable ('info' by default).
 
 Logging level can be changed in runtime with
 [standard zap handler](https://github.com/uber-go/zap/blob/v1.23.0/http_handler.go#L33-L70)
 exposed at `/log/level`.
 
-### Actions debugging
-
+## Actions debugging
 To debug any working action you must enable http server via '-http' flag
 and visit an endpoint `/pipelines/<pipeline_name>/<plugin_index_in_config>/sample`.
 It will show 1 sample of the in/out events.
@@ -46,8 +43,7 @@ If `-http=':9090'` debug endpoints will be:
 
 > **Note**: by default debug server starts on address `:9000` (can be checked using `--help` flag). If your server uses IPv6, the debug server with address configured as `:<port>` will start on IPv6 port.
 
-### Overriding configurations
-
+## Overriding configurations
 You can use multiple configuration files. This allows you to define a base configuration (e.g., common.yaml) and override or extend it with additional configurations (e.g., local.yaml).
 
 ```
@@ -88,8 +84,7 @@ pipelines:
 
 Arrays (or lists) are usually replaced entirely when merging configurations (e.g., actions). Dictionaries (or maps), on the other hand, are typically merged (e.g., output.type).
 
-### Overriding by environment variables
-
+## Overriding by environment variables
 `file.d` can override config fields if you specify environment variables with `FILED_` prefix.  
 The name of the env will be divided by underscores and the config will set or override the config field by the resulted
 path.
@@ -107,14 +102,12 @@ pipelines:
   pipeline_name: ...
 ```
 
-### Vault support
-
+## Vault support
 Consider this config:
-
 ```yaml
 vault:
-  token: example_token
   address: http://127.0.0.1:8200
+  token: example_token
 pipelines:
   example:
     input:
@@ -124,14 +117,27 @@ pipelines:
       type: devnull
 ```
 
-`file.d` supports getting secrets from Vault as soon as you specify Vault token and an address in a configuration.  
-Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`,  
-and `file.d` tries to connect to Vault and get the secret from there.  
-If you need to pass a literal string that begins with `vault(`, you should escape the value with a
-backslash: `\vault(path/to/secret, key)`.
+`file.d` supports getting secrets from Vault as soon as you specify Vault address and one of auth methods in a configuration.
+Then you can write any field-string in both arrays and dictionaries using syntax `vault(path/to/secret, key)`, and `file.d` tries to connect to Vault and get the secret from there.
 
-### Env support
+> If you need to pass a literal string that begins with `vault(`, you should escape the value with a backslash: `\vault(path/to/secret, key)`.
 
+### Auth methods
+Token:
+```yaml
+vault:
+  token: example_token
+```
+
+AppRole:
+```yaml
+vault:
+  role_id: example_role
+  secret_id: example_secret
+  auth_mount_path: some/path # used when formatting the authorization uri: 'auth/%s/login'
+```
+
+## Env support
 Consider this config:
 
 ```yaml
@@ -146,16 +152,14 @@ pipelines:
         field: env(ENV_NAME)
 ```
 
-`file.d` supports getting environment variables. Then you can write any
-field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`,  
-and `file.d` tries to get environment variable value. If you need to pass
-a literal string that begins with `env(`, you should escape the value with a
+`file.d` supports getting environment variables. Then you can write any field-string in both arrays and dictionaries using syntax `env(ENV_NAME)`, and `file.d` tries to get environment variable value.
+
+> If you need to pass a literal string that begins with `env(`, you should escape the value with a
 backslash: `\env(ENV_NAME)`.
 
-### Do action if match
+## Do action if match
 
 ### match_fields
-
 File.d can do any action if it matches by pattern.
 In the `match_fields` you can pass some patterns, for example:
 
@@ -225,8 +229,7 @@ Patterns must have a list ([]) or string type, not a number or null.
 ### Match modes
 @match-modes|header-description
 
-### Decoders
-
+## Decoders
 If you have logs in specific non-json format, you can specify decoder type in pipeline settings. By default `json` decoder is used. More details can be found [here](/decoder/readme.md).
 
 ```yml