From f189c2858078413dc9b8db5877adaa9dba173f4f Mon Sep 17 00:00:00 2001 From: Sascha Wolke Date: Thu, 30 Jan 2025 18:32:31 +0100 Subject: [PATCH] logging: support JSON output in the default logger Add a new "format" option to the log section of the configuration file, that supports text and JSON based output in the default logger. The text format will be used if no format was specified, making the option backward compatible. Signed-off-by: Sascha Wolke --- cmd/kes/server.go | 10 ++++---- internal/log/format.go | 17 +++++++++++++ kesconf/config.go | 22 +++++++++++++++-- kesconf/config_test.go | 32 +++++++++++++++++++++++++ kesconf/file.go | 4 ++++ kesconf/testdata/log-format-default.yml | 24 +++++++++++++++++++ kesconf/testdata/log-format-json.yml | 25 +++++++++++++++++++ log.go | 12 ++++++++++ server.go | 8 +++++-- 9 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 internal/log/format.go create mode 100644 kesconf/testdata/log-format-default.yml create mode 100644 kesconf/testdata/log-format-json.yml diff --git a/cmd/kes/server.go b/cmd/kes/server.go index 1e640766..bc7aff9d 100644 --- a/cmd/kes/server.go +++ b/cmd/kes/server.go @@ -183,6 +183,7 @@ func startServer(addrFlag, configFlag string) error { srv := &kes.Server{} conf.Cache = configureCache(conf.Cache) if rawConfig.Log != nil { + srv.LogFormat = rawConfig.Log.LogFormat srv.ErrLevel.Set(rawConfig.Log.ErrLevel) srv.AuditLevel.Set(rawConfig.Log.AuditLevel) } @@ -215,9 +216,9 @@ func startServer(addrFlag, configFlag string) error { } else { fmt.Fprintf(buf, "%-33s \n", blue.Render("Admin")) } - fmt.Fprintf(buf, "%-33s error=stderr level=%s\n", blue.Render("Logs"), srv.ErrLevel.Level()) + fmt.Fprintf(buf, "%-33s error=stderr level=%s format=%s\n", blue.Render("Logs"), srv.ErrLevel.Level(), srv.LogFormat) if srv.AuditLevel.Level() <= slog.LevelInfo { - fmt.Fprintf(buf, "%-11s audit=stdout level=%s\n", " ", srv.AuditLevel.Level()) + fmt.Fprintf(buf, "%-11s audit=stdout level=%s format=%s\n", " ", srv.AuditLevel.Level(), srv.LogFormat) } if memLocked { fmt.Fprintf(buf, "%-33s %s\n", blue.Render("MLock"), "enabled") @@ -251,6 +252,7 @@ func startServer(addrFlag, configFlag string) error { continue } if file.Log != nil { + srv.LogFormat = file.Log.LogFormat srv.ErrLevel.Set(file.Log.ErrLevel) srv.AuditLevel.Set(file.Log.AuditLevel) } @@ -375,8 +377,8 @@ func startDevServer(addr string) error { fmt.Fprintln(buf) fmt.Fprintf(buf, "%-33s %s\n", blue.Render("API Key"), apiKey.String()) fmt.Fprintf(buf, "%-33s %s\n", blue.Render("Admin"), apiKey.Identity()) - fmt.Fprintf(buf, "%-33s error=stderr level=%s\n", blue.Render("Logs"), srv.ErrLevel.Level()) - fmt.Fprintf(buf, "%-11s audit=stdout level=%s\n", " ", srv.AuditLevel.Level()) + fmt.Fprintf(buf, "%-33s error=stderr level=%s format=%s\n", blue.Render("Logs"), srv.ErrLevel.Level(), srv.LogFormat) + fmt.Fprintf(buf, "%-11s audit=stdout level=%s format=%s\n", " ", srv.AuditLevel.Level(), srv.LogFormat) fmt.Fprintln(buf) fmt.Fprintln(buf, "=> Server is up and running...") fmt.Println(buf.String()) diff --git a/internal/log/format.go b/internal/log/format.go new file mode 100644 index 00000000..f4e29a57 --- /dev/null +++ b/internal/log/format.go @@ -0,0 +1,17 @@ +// Copyright 2025 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package log + +// Format defines a type of different log output formats, +// used by audit and error events if no custom log handler specified. +type Format string + +const ( + // TextFormat creates plain text formatted log message + TextFormat Format = "Text" + + // JSONFormat creates JSON formatted log messages + JSONFormat Format = "JSON" +) diff --git a/kesconf/config.go b/kesconf/config.go index 67860da2..2c1873ff 100644 --- a/kesconf/config.go +++ b/kesconf/config.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/minio/kes/internal/log" "github.com/minio/kms-go/kes" "gopkg.in/yaml.v3" ) @@ -64,8 +65,9 @@ type ymlFile struct { } `yaml:"api"` Log struct { - Error env[string] `yaml:"error"` - Audit env[string] `yaml:"audit"` + Format env[string] `yaml:"format"` + Error env[string] `yaml:"error"` + Audit env[string] `yaml:"audit"` } `yaml:"log"` Keys []struct { @@ -291,6 +293,10 @@ func ymlToServerConfig(y *ymlFile) (*File, error) { return nil, fmt.Errorf("kesconf: invalid offline cache expiry '%v'", y.Cache.Expiry.Offline.Value) } + logFormat, err := parseLogFormat(y.Log.Format.Value) + if err != nil { + return nil, err + } errLevel, err := parseLogLevel(y.Log.Error.Value) if err != nil { return nil, err @@ -352,6 +358,7 @@ func ymlToServerConfig(y *ymlFile) (*File, error) { ExpiryOffline: y.Cache.Expiry.Offline.Value, }, Log: &LogConfig{ + LogFormat: logFormat, ErrLevel: errLevel, AuditLevel: auditLevel, }, @@ -701,6 +708,17 @@ func (r *env[T]) UnmarshalYAML(node *yaml.Node) error { return nil } +func parseLogFormat(s string) (log.Format, error) { + switch strings.TrimSpace(strings.ToUpper(s)) { + case "JSON": + return log.JSONFormat, nil + case "TEXT", "": + return log.TextFormat, nil + default: + return log.TextFormat, fmt.Errorf("log format: unknown format '%s'", s) + } +} + func parseLogLevel(s string) (slog.Level, error) { const ( LevelDebug = "DEBUG" diff --git a/kesconf/config_test.go b/kesconf/config_test.go index 4531a62a..56db86ad 100644 --- a/kesconf/config_test.go +++ b/kesconf/config_test.go @@ -7,6 +7,8 @@ package kesconf import ( "testing" "time" + + "github.com/minio/kes/internal/log" ) func TestReadServerConfigYAML_FS(t *testing.T) { @@ -70,6 +72,36 @@ func TestReadServerConfigYAML_CustomAPI(t *testing.T) { } } +func TestReadServerConfigYAML_DefaultLogFormat(t *testing.T) { + const ( + Filename = "./testdata/log-format-default.yml" + ) + + config, err := ReadFile(Filename) + if err != nil { + t.Fatalf("Failed to read file '%s': %v", Filename, err) + } + + if config.Log.LogFormat != log.TextFormat { + t.Fatalf("Invalid log config: invalid format: got '%s' - want '%s'", config.Log.LogFormat, log.TextFormat) + } +} + +func TestReadServerConfigYAML_JSONLogFormat(t *testing.T) { + const ( + Filename = "./testdata/log-format-json.yml" + ) + + config, err := ReadFile(Filename) + if err != nil { + t.Fatalf("Failed to read file '%s': %v", Filename, err) + } + + if config.Log.LogFormat != log.JSONFormat { + t.Fatalf("Invalid log config: invalid format: got '%s' - want '%s'", config.Log.LogFormat, log.JSONFormat) + } +} + func TestReadServerConfigYAML_VaultWithAppRole(t *testing.T) { const ( Filename = "./testdata/vault-approle.yml" diff --git a/kesconf/file.go b/kesconf/file.go index c60218d6..9ed3bf53 100644 --- a/kesconf/file.go +++ b/kesconf/file.go @@ -28,6 +28,7 @@ import ( "github.com/minio/kes/internal/keystore/gcp" "github.com/minio/kes/internal/keystore/gemalto" "github.com/minio/kes/internal/keystore/vault" + "github.com/minio/kes/internal/log" kesdk "github.com/minio/kms-go/kes" yaml "gopkg.in/yaml.v3" ) @@ -291,6 +292,9 @@ type CacheConfig struct { // LogConfig is a structure that holds the logging configuration // for a KES server. type LogConfig struct { + // LogFormat determines the format of the default audit and error logger. + LogFormat log.Format + // Error determines whether the KES server logs error events to STDERR. // It does not en/disable error logging in general. ErrLevel slog.Level diff --git a/kesconf/testdata/log-format-default.yml b/kesconf/testdata/log-format-default.yml new file mode 100644 index 00000000..1543657d --- /dev/null +++ b/kesconf/testdata/log-format-default.yml @@ -0,0 +1,24 @@ + +address: 0.0.0.0:7373 +admin: + identity: disabled + +tls: + key: ./private.key + cert: ./public.crt + +cache: + expiry: + any: 5m0s + unused: 30s + offline: 0s + +keystore: + fs: + path: /tmp/kes + +# no log format in this file, use defaults +log: + error: INFO + audit: INFO + diff --git a/kesconf/testdata/log-format-json.yml b/kesconf/testdata/log-format-json.yml new file mode 100644 index 00000000..5b48e59d --- /dev/null +++ b/kesconf/testdata/log-format-json.yml @@ -0,0 +1,25 @@ + +address: 0.0.0.0:7373 +admin: + identity: disabled + +tls: + key: ./private.key + cert: ./public.crt + +cache: + expiry: + any: 5m0s + unused: 30s + offline: 0s + +keystore: + fs: + path: /tmp/kes + +# use JSON output format +log: + format: JSON + error: INFO + audit: INFO + diff --git a/log.go b/log.go index 0dcfc4cb..1eb1e31d 100644 --- a/log.go +++ b/log.go @@ -6,9 +6,11 @@ package kes import ( "context" + "io" "log/slog" "github.com/minio/kes/internal/api" + "github.com/minio/kes/internal/log" ) // logHandler is an slog.Handler that handles Server log records. @@ -45,6 +47,16 @@ func newLogHandler(h slog.Handler, level slog.Leveler) *logHandler { return handler } +// newFormattedLogHandler returns a new text or JSON formatted log handler. +func newFormattedLogHandler(w io.Writer, f log.Format, opts *slog.HandlerOptions) slog.Handler { + switch f { + case log.JSONFormat: + return slog.NewJSONHandler(w, opts) + default: + return slog.NewTextHandler(w, opts) + } +} + // Enabled reports whether h handles records at the given level. func (h *logHandler) Enabled(ctx context.Context, level slog.Level) bool { return level >= h.level.Level() && h.h.Enabled(ctx, level) || diff --git a/server.go b/server.go index 8f16cee6..418e978c 100644 --- a/server.go +++ b/server.go @@ -29,6 +29,7 @@ import ( "github.com/minio/kes/internal/headers" "github.com/minio/kes/internal/https" "github.com/minio/kes/internal/keystore" + "github.com/minio/kes/internal/log" "github.com/minio/kes/internal/metric" "github.com/minio/kes/internal/sys" "github.com/minio/kms-go/kes" @@ -56,6 +57,9 @@ type Server struct { // connections to return to idle and then shut down. ShutdownTimeout time.Duration + // LogFormat controls the output format of the default logger. + LogFormat log.Format + // ErrLevel controls which errors are logged by the server. // It may be adjusted after the server has been started to // change its logging behavior. @@ -410,7 +414,7 @@ func (s *Server) listen(ctx context.Context, ln net.Listener, conf *Config) (net if conf.ErrorLog == nil { state.LogHandler = newLogHandler( - slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + newFormattedLogHandler(os.Stderr, s.LogFormat, &slog.HandlerOptions{ Level: &s.ErrLevel, }), &s.ErrLevel, @@ -421,7 +425,7 @@ func (s *Server) listen(ctx context.Context, ln net.Listener, conf *Config) (net state.Log = slog.New(state.LogHandler) if conf.AuditLog == nil { - handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: &s.AuditLevel}) + handler := newFormattedLogHandler(os.Stdout, s.LogFormat, &slog.HandlerOptions{Level: &s.AuditLevel}) state.Audit = newAuditLogger(&AuditLogHandler{Handler: handler}, &s.AuditLevel) } else { state.Audit = newAuditLogger(conf.AuditLog, &s.AuditLevel)