diff --git a/veinmind-runner/cmd/cli.go b/veinmind-runner/cmd/cli.go index 26f723b9..7d749aa0 100644 --- a/veinmind-runner/cmd/cli.go +++ b/veinmind-runner/cmd/cli.go @@ -164,6 +164,38 @@ var authCmd = &cmd.Command{ return runner.Run() }, } + +var webhookCmd = &cmd.Command{ + Use: "webhook", + Short: "webhook for harbor", + RunE: func(cmd *cobra.Command, args []string) error { + path, err := cmd.Flags().GetString("config") + if err != nil { + return err + } + + config, err := authz.NewHarborWebhookConfig(path) + if err != nil { + return err + } + + options := []authz.HarborWebhookOption{ + authz.WithHarborPolicy(config.Policies...), + authz.WithHarborAuthLog(config.Log.AuthZLogPath), + authz.WithHarborPluginLog(config.Log.PluginLogPath), + authz.WithWebhookServer(config.WebhookServer), + authz.WithAuthInfo(config.DockerAuth), + authz.WithMailServer(config.MailConf), + } + + server, err := authz.NewHarborWebhookServer(options...) + if err != nil { + return err + } + runner := authz.NewDefaultRunner(&server) + return runner.Run() + }, +} var listCmd = &cmd.Command{ Use: "list", Short: "list relevant information", @@ -230,7 +262,7 @@ var scanRegistryCmd = &cmd.Command{ if config == "" { c, err = commonRuntime.NewDockerClient() } else { - c, err = commonRuntime.NewDockerClient(commonRuntime.WithAuth(config)) + c, err = commonRuntime.NewDockerClient(commonRuntime.WithAuthFromPath(config)) } if err != nil { return err @@ -471,6 +503,8 @@ func init() { rootCmd.AddCommand(scanRegistryCmd) rootCmd.AddCommand(authCmd) authCmd.Flags().StringP("config", "c", "", "authz config path") + rootCmd.AddCommand(webhookCmd) + webhookCmd.Flags().StringP("config", "c", "", "webhook config path") rootCmd.AddCommand(listCmd) rootCmd.PersistentFlags().IntP("exit-code", "e", 0, "exit-code when veinmind-runner find security issues") listCmd.AddCommand(listPluginCmd) diff --git a/veinmind-runner/go.mod b/veinmind-runner/go.mod index b57fdccd..af5573a1 100644 --- a/veinmind-runner/go.mod +++ b/veinmind-runner/go.mod @@ -5,13 +5,15 @@ go 1.16 require ( github.com/BurntSushi/toml v0.4.1 github.com/chaitin/libveinmind v1.1.1 - github.com/chaitin/veinmind-common-go v1.0.5 + github.com/chaitin/veinmind-common-go v1.1.0 github.com/distribution/distribution v2.8.1+incompatible github.com/docker/docker v20.10.17+incompatible github.com/gin-gonic/gin v1.8.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.4.0 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/mail.v2 v2.3.1 gotest.tools/v3 v3.1.0 // indirect ) diff --git a/veinmind-runner/go.sum b/veinmind-runner/go.sum index fa4afd0c..330b1bf1 100644 --- a/veinmind-runner/go.sum +++ b/veinmind-runner/go.sum @@ -203,8 +203,8 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chaitin/libveinmind v1.1.1 h1:DoMJXAjw3xzOHcrAiBBQpbqudsumw5pGfRQko6f6USc= github.com/chaitin/libveinmind v1.1.1/go.mod h1:bUUjhkyZyZ9sTetpm5rOfj5TU3hr5moE3VQM+IgHrbw= -github.com/chaitin/veinmind-common-go v1.0.5 h1:OA8c9IDMPGXUBQksiFck6tx5eEtfDz9LB1+FcmaSMDY= -github.com/chaitin/veinmind-common-go v1.0.5/go.mod h1:pmtVj6duS6B3spaBPhMMXOZOzDyyABtcEIQEF4KpZaI= +github.com/chaitin/veinmind-common-go v1.1.0 h1:YKE+KBVyP48IBKFf4dH5Ve8ayHz8rQ8S07iFPF8z60U= +github.com/chaitin/veinmind-common-go v1.1.0/go.mod h1:Ap6KTM2qqKv+8tLeb38pX9DFWt/P8/1gmCO2b9tjNZo= github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af/go.mod h1:Qjyv4H3//PWVzTeCezG2b9IRn6myJxJSr4TD/xo6ojU= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= @@ -2047,6 +2047,8 @@ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscL google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2069,6 +2071,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= +gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM= diff --git a/veinmind-runner/pkg/authz/action/harbor_webhook.go b/veinmind-runner/pkg/authz/action/harbor_webhook.go new file mode 100644 index 00000000..2ac99ba1 --- /dev/null +++ b/veinmind-runner/pkg/authz/action/harbor_webhook.go @@ -0,0 +1,29 @@ +package action + +var ( + PULL_ARTIFACT string = "PULL_ARTIFACT" + PUSH_ARTIFACT string = "PUSH_ARTIFACT" + DELETE_ARTIFACT string = "DELETE_ARTIFACT" +) + +// only for push and pull api +// TODO: parse scan/helm/etc. api post data +type PullandPushData struct { + Type string `json:"type"` + OccurAt int `json:"occur_at"` + Operator string `json:"operator"` + EventData struct { + Resources []struct { + Digest string `json:"digest"` + Tag string `json:"tag"` + ResourceURL string `json:"resource_url"` + } `json:"resources"` + Repository struct { + DateCreated int `json:"date_created"` + Name string `json:"name"` + Namespace string `json:"namespace"` + RepoFullName string `json:"repo_full_name"` + RepoType string `json:"repo_type"` + } `json:"repository"` + } `json:"event_data"` +} diff --git a/veinmind-runner/pkg/authz/authz_config.go b/veinmind-runner/pkg/authz/authz_config.go index 4a78f8fa..4cf8b95a 100644 --- a/veinmind-runner/pkg/authz/authz_config.go +++ b/veinmind-runner/pkg/authz/authz_config.go @@ -14,6 +14,8 @@ const ( defaultSockListenAddr = "/run/docker/plugins/veinmind-broker.sock" ) + + type Policy struct { Action string `toml:"action"` EnabledPlugins []string `toml:"enabled_plugins"` diff --git a/veinmind-runner/pkg/authz/authz_report.go b/veinmind-runner/pkg/authz/authz_report.go index da98a474..86f713e3 100644 --- a/veinmind-runner/pkg/authz/authz_report.go +++ b/veinmind-runner/pkg/authz/authz_report.go @@ -4,6 +4,8 @@ import ( "fmt" "io" + gomail "gopkg.in/mail.v2" + "github.com/chaitin/libveinmind/go/plugin/log" "github.com/chaitin/veinmind-common-go/service/report" "github.com/chaitin/veinmind-tools/veinmind-runner/pkg/reporter" @@ -54,3 +56,34 @@ func handleDockerPluginReportEvents(eventListCh <-chan []reporter.ReportEvent, b log.Warn(err) } } + +func handleHarborWebhookReportEvents(eventListCh <-chan []reporter.ReportEvent, hpolicy HarborPolicy, + pluginLog io.Writer, mailconf MailConf) { + filter, events := processReportEvents(eventListCh, hpolicy.Policy, pluginLog) + if filter { + if hpolicy.Alert { + log.Warn(fmt.Sprintf("Action %s has risks!", hpolicy.Action)) + } + if hpolicy.SendMail { + err := sendReport2Mail(events, mailconf) + if err != nil { + log.Error(err) + } + } + } + if err := reporter.WriteEvents2Log(events, pluginLog); err != nil { + log.Warn(err) + } +} +func sendReport2Mail(events []reporter.ReportEvent, mailconf MailConf) error { + d := gomail.NewDialer(mailconf.Host, mailconf.Port, mailconf.Name, mailconf.Password) + m := gomail.NewMessage() + m.SetHeader("From", mailconf.Name) + m.SetHeader("To", mailconf.Subscriber...) + m.SetHeader("Subject", "Harbor webhook Report") + m.SetBody("text/plain", fmt.Sprintf("%#v", events)) + if err := d.DialAndSend(m); err != nil { + return err + } + return nil +} diff --git a/veinmind-runner/pkg/authz/harbor_webhook.go b/veinmind-runner/pkg/authz/harbor_webhook.go new file mode 100644 index 00000000..f0ec2000 --- /dev/null +++ b/veinmind-runner/pkg/authz/harbor_webhook.go @@ -0,0 +1,295 @@ +package authz + +import ( + "context" + "encoding/json" + "errors" + "io" + "io/ioutil" + "os" + "strconv" + "sync" + + "github.com/chaitin/libveinmind/go/plugin/log" + "github.com/chaitin/veinmind-common-go/pkg/auth" + "github.com/chaitin/veinmind-common-go/runtime" + "github.com/chaitin/veinmind-tools/veinmind-runner/pkg/authz/action" + "github.com/chaitin/veinmind-tools/veinmind-runner/pkg/reporter" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" +) + +type HarborWebhookOption func(hwopt *harborWebhookOption) error + +type harborWebhookOption struct { + authLog io.WriteCloser + pluginLog io.WriteCloser + policies sync.Map + authInfo auth.Auth + mailConf MailConf + WebhookServer WebhookServer +} + +func WithMailServer(mailconf MailConf) HarborWebhookOption { + return func(hwopt *harborWebhookOption) error { + hwopt.mailConf = mailconf + return nil + } +} +func WithAuthInfo(auth auth.Auth) HarborWebhookOption { + return func(hwopt *harborWebhookOption) error { + hwopt.authInfo = auth + return nil + } +} + +func WithHarborPolicy(hwpolicies ...HarborPolicy) HarborWebhookOption { + return func(option *harborWebhookOption) error { + for _, hwpolicy := range hwpolicies { + option.policies.Store(hwpolicy.Action, hwpolicy) + } + + return nil + } +} + +func WithHarborAuthLog(path string) HarborWebhookOption { + return func(option *harborWebhookOption) error { + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + _, err = os.Create(path) + if err != nil { + return err + } + } + + fp, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return err + } + + option.authLog = fp + return nil + } +} + +func WithHarborPluginLog(path string) HarborWebhookOption { + return func(option *harborWebhookOption) error { + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + _, err = os.Create(path) + if err != nil { + return err + } + } + + fp, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return err + } + + option.pluginLog = fp + return nil + } +} + +func WithWebhookServer(webhookServer WebhookServer) HarborWebhookOption { + return func(hwopt *harborWebhookOption) error { + hwopt.WebhookServer = webhookServer + return nil + } +} +func WithHarborServerOptions(options ...HarborWebhookOption) HarborWebhookOption { + return func(s *harborWebhookOption) error { + for _, option := range options { + if err := option(s); err != nil { + return err + } + } + return nil + } +} + +type harborWebhookServer struct { + defaultServer + hwoption *harborWebhookOption +} + +func NewHarborWebhookServer(options ...HarborWebhookOption) (harborWebhookServer, error) { + option := &harborWebhookOption{} + for _, opt := range options { + if err := opt(option); err != nil { + return harborWebhookServer{}, err + } + } + return harborWebhookServer{hwoption: option}, nil +} + +func (s *harborWebhookServer) Init() error { + opts := make([]HarborWebhookOption, 0) + if s.hwoption.authLog == nil { + opts = append(opts, WithHarborAuthLog(defaultAuthLogPath)) + } + if s.hwoption.pluginLog == nil { + opts = append(opts, WithHarborPluginLog(defaultPluginPath)) + } + if s.hwoption.WebhookServer == (WebhookServer{}) { + opts = append(opts, WithWebhookServer(defaultWebHookServer)) + } + if err := WithHarborServerOptions(opts...)(s.hwoption); err != nil { + return err + } + return nil +} + +func (s *harborWebhookServer) Start() error { + multiWriter := io.MultiWriter(s.hwoption.authLog, os.Stdout) + + logger := logrus.New() + logger.Out = multiWriter + + log.SetDefaultLogger(log.NewLogrus(logger)) + gin.DefaultWriter = multiWriter + + engine := s.registerRouter() + + go func() { + port := ":" + strconv.Itoa(s.hwoption.WebhookServer.Port) + err := engine.Run(port) + if err != nil { + log.Error(err) + } + }() + return nil +} + +func (s *harborWebhookServer) Close() error { + err := s.hwoption.authLog.Close() + if err != nil { + return err + } + + err = s.hwoption.pluginLog.Close() + if err != nil { + return err + } + return nil +} + +func (s *harborWebhookServer) registerRouter() *gin.Engine { + engine := gin.Default() + engine.POST("/api", func(c *gin.Context) { + apiHandler(c, *s.hwoption) + }) + // //get post data content from this url + // engine.POST("/", func(c *gin.Context) { + // var body map[string]interface{} + // data, _ := ioutil.ReadAll(c.Request.Body) + // if err := json.Unmarshal(data, &body); err != nil { + // fmt.Println(err) + // } + // fmt.Println("body data => ", string(data)) + // for k, v := range c.Request.Header { + // fmt.Println(k, v) + // } + // c.JSON(http.StatusOK, struct{}{}) + // }) + return engine +} + +func apiHandler(c *gin.Context, option harborWebhookOption) { + if err := checkPassword(c, option.WebhookServer.Authorization); err != nil { + log.Error(err) + return + } + + postData, err := parseHarborwebhookPostdata(c) + if err != nil { + log.Error(err) + return + } + + if postData.Operator == "webhook" { + return + } + imageNames, err := getImageNames(postData) + if err != nil { + log.Error(err) + return + } + err = pullImagesFromHarbor(option.authInfo, imageNames) + if err != nil { + log.Error(err) + return + } + val, ok := option.policies.Load(postData.Type) + if !ok { + log.Error(err) + return + } + hpolicy := val.(HarborPolicy) + var eventListCh chan []reporter.ReportEvent + switch postData.Type { + case action.PUSH_ARTIFACT: + eventListCh, err = HandleWebhookImagePush(context.Background(), hpolicy.Policy, imageNames) + // TODO: other type's process + default: + return + } + if err != nil { + return + } + go func() { + handleHarborWebhookReportEvents(eventListCh, hpolicy, + option.pluginLog, option.mailConf) + }() +} + +// get secrect from Authorization field and check +func checkPassword(c *gin.Context, password string) error { + if password == "" { + return nil + } + if c.Request.Header.Get("Authorization") == password { + return nil + } + return errors.New("error passowrd") +} + +func parseHarborwebhookPostdata(c *gin.Context) (action.PullandPushData, error) { + postData := &action.PullandPushData{} + data, _ := ioutil.ReadAll(c.Request.Body) + if err := json.Unmarshal(data, &postData); err != nil { + return action.PullandPushData{}, err + } + return *postData, nil +} +func getImageNames(data action.PullandPushData) ([]string, error) { + resources := data.EventData.Resources + if len(resources) < 1 { + return []string{}, errors.New("no image choosed") + } + var imagenames []string + for _, resource := range resources { + imagenames = append(imagenames, resource.ResourceURL) + } + return imagenames, nil +} + +// download relevant images +func pullImagesFromHarbor(authentity auth.Auth, imageNames []string) error { + authConfig := auth.AuthConfig{ + Auths: []auth.Auth{authentity}} + dockerclient, err := runtime.NewDockerClient(runtime.WithAuth(authConfig)) + if err != nil { + return err + } + for _, img := range imageNames { + _, err := dockerclient.Pull(img) + if err != nil { + log.Error(err) + continue + } + } + return nil +} diff --git a/veinmind-runner/pkg/authz/harbor_webhook_auth.go b/veinmind-runner/pkg/authz/harbor_webhook_auth.go new file mode 100644 index 00000000..fcfa9212 --- /dev/null +++ b/veinmind-runner/pkg/authz/harbor_webhook_auth.go @@ -0,0 +1,25 @@ +package authz + +import ( + "context" + + "github.com/chaitin/libveinmind/go/plugin/log" + "github.com/chaitin/veinmind-tools/veinmind-runner/pkg/reporter" + "github.com/chaitin/veinmind-tools/veinmind-runner/pkg/scan" +) + +func HandleWebhookImagePush(ctx context.Context, policy Policy, imageNames []string) (chan []reporter.ReportEvent, error) { + eventListCh := make(chan []reporter.ReportEvent, 1) + var result []reporter.ReportEvent + for _, img := range imageNames { + report, err := scan.ScanLocalImage(ctx, img, + policy.EnabledPlugins, policy.PluginParams) + if err != nil { + log.Error(err) + continue + } + result = append(result, report...) + } + eventListCh <- result + return eventListCh, nil +} diff --git a/veinmind-runner/pkg/authz/harbor_webhook_config.go b/veinmind-runner/pkg/authz/harbor_webhook_config.go new file mode 100644 index 00000000..2c19726b --- /dev/null +++ b/veinmind-runner/pkg/authz/harbor_webhook_config.go @@ -0,0 +1,57 @@ +package authz + +import ( + "errors" + + "github.com/BurntSushi/toml" + "github.com/chaitin/veinmind-common-go/pkg/auth" +) + +var ( + defaultWebHookServer = WebhookServer{Port: 8080} +) + +type MailConf struct { + Host string `toml:"host"` + Port int `toml:"port"` + Name string `toml:"username"` + Password string `toml:"password"` + Subscriber []string `toml:"subscriber"` +} + +type HarborPolicy struct { + Policy + SendMail bool `toml:"send_mail"` +} + +type WebhookServer struct { + Port int `toml:"port"` + Authorization string `toml:"authorization"` +} + +type HarborWebhookConfig struct { + Log Log `toml:"log"` + WebhookServer WebhookServer `toml:"webhook_server"` + DockerAuth auth.Auth `toml:"docker_auth"` + Policies []HarborPolicy `toml:"policies"` + MailConf MailConf `toml:"mail_conf"` +} + +func NewHarborWebhookConfig(paths ...string) (*HarborWebhookConfig, error) { + if len(paths) < 1 { + return nil, errors.New("config path can't be empty") + } + + path := defaultConfigPath + if paths[0] != "" { + path = paths[0] + } + + result := &HarborWebhookConfig{} + _, err := toml.DecodeFile(path, result) + if err != nil { + return nil, err + } + + return result, nil +}