diff --git a/.github/scripts/components-scripts/conformance-configuration.git-setup.sh b/.github/scripts/components-scripts/conformance-configuration.git-setup.sh new file mode 100755 index 0000000000..033237d7e8 --- /dev/null +++ b/.github/scripts/components-scripts/conformance-configuration.git-setup.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Set up a local bare git upstream used by the configuration.git conformance +# and certification tests. Exports GIT_REMOTE_URL pointing at a file:// URL +# so the tests don't need any network access. + +set -e + +ROOT=/tmp/dapr-conf-git +rm -rf "$ROOT" +mkdir -p "$ROOT" + +git init --bare "$ROOT/upstream.git" + +SEED="$ROOT/seed" +git clone "$ROOT/upstream.git" "$SEED" +git -C "$SEED" config user.email "ci@dapr.io" +git -C "$SEED" config user.name "ci" +git -C "$SEED" config commit.gpgsign false +git -C "$SEED" commit --allow-empty -m "initial" +git -C "$SEED" branch -M main +git -C "$SEED" push origin main + +# Point the bare repo's HEAD at refs/heads/main. Without this, HEAD remains +# the default `refs/heads/master` (which doesn't exist), and go-git's +# PlainClone treats the upstream as effectively empty — the consumer then +# falls into a fresh-init path, producing a non-shared-history clone that +# can't fast-forward push back. +git --git-dir "$ROOT/upstream.git" symbolic-ref HEAD refs/heads/main + +echo "GIT_REMOTE_URL=file://$ROOT/upstream.git" >> "$GITHUB_ENV" diff --git a/.github/scripts/test-info.mjs b/.github/scripts/test-info.mjs index 79ecc1802d..257b82e98a 100644 --- a/.github/scripts/test-info.mjs +++ b/.github/scripts/test-info.mjs @@ -279,6 +279,16 @@ const components = { conformanceSetup: 'conformance-configuration.kubernetes-setup.sh', sourcePkg: ['configuration/kubernetes'], }, + 'configuration.git': { + certification: true, + certificationSetup: 'conformance-configuration.git-setup.sh', + sourcePkg: ['configuration/git'], + }, + 'configuration.git.local': { + conformance: true, + conformanceSetup: 'conformance-configuration.git-setup.sh', + sourcePkg: ['configuration/git'], + }, 'crypto.azure.keyvault': { conformance: true, requiredSecrets: [ diff --git a/.golangci.yml b/.golangci.yml index 3ce398fb7f..c0d7616b94 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -162,6 +162,7 @@ linters: misspell: ignore-rules: - someword + - prompty lll: line-length: 120 tab-width: 1 diff --git a/configuration/git/README.md b/configuration/git/README.md new file mode 100644 index 0000000000..8be21f7cb0 --- /dev/null +++ b/configuration/git/README.md @@ -0,0 +1,241 @@ +# Git Configuration Store + +A Dapr configuration store backed by a git repository. The store clones the +configured repo on `Init`, polls the upstream for new commits at a configurable +interval, and notifies subscribers when keys change. + +> **Status:** alpha. Requires a paired `dapr/dapr` PR registering +> `configuration.git` (e.g. `cmd/daprd/components/configuration_git.go`) +> before it is usable from a sidecar. + +## When to use it + +The component is intended for **operator-driven configuration repos** +(prompts, instructions, agent definitions, feature flags) where: + +- The data is small (≤ 1 MiB per file by default). +- Changes are infrequent (commits, not high-frequency writes). +- A polling delay of a few minutes is acceptable. + +It is **not** suitable for source-code monorepos or hot-path data planes. + +## Quick start + +```yaml +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: configstore +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "https://github.com/example/agent-config.git" + - name: branch + value: "main" + - name: mappingMode + value: "file" + - name: pollInterval + value: "5m" +``` + +## Mapping modes + +The `mappingMode` metadata field selects how files in the repository become +configuration items. Matching is case-insensitive. **Non-matching files in +the configured scope are a hard error** — if the directory contains a mix of +file types, either narrow `path` to the homogeneous subset or use +`mappingMode: file`. + +The `agentYaml` and `prompty` modes target dapr-agents-style configuration +repos. `prompty` follows the [Prompty spec](https://github.com/microsoft/prompty) +for `.prompty` files; `agentYaml` is a convenience mode for repos that store +agent definitions as YAML/JSON (the name describes the intended layout, not +an external spec). + +### `file` (default) + +Each file becomes one config item. The relative POSIX path is the key, +the file contents the value. + +``` +repo/ + agents/weather/agent_role.txt → key "agents/weather/agent_role.txt" + agents/weather/agent_goal.txt → key "agents/weather/agent_goal.txt" +``` + +Recommended when the consumer (e.g. dapr-agents) expects scalar config keys. + +### `agentYaml` + +A convenience mode for repos that store agent definitions as YAML/JSON. +Each `*.yaml`, `*.yml`, or `*.json` file is parsed as a flat top-level map. +Each top-level field becomes a key prefixed by the filename stem with +directory separators replaced by `_`. **Non-YAML/JSON files in scope cause +Init to fail.** + +```yaml +# repo/agents/weather.yaml +agent_role: Weather expert +agent_goal: Help users plan trips +agent_instructions: + - be concise + - cite sources +``` + +produces keys: + +``` +agents_weather/agent_role = "Weather expert" +agents_weather/agent_goal = "Help users plan trips" +agents_weather/agent_instructions = (YAML-serialised list — round-trip via yaml.Unmarshal) +``` + +### `prompty` + +Each `*.prompty` file's YAML frontmatter and body are split. +Frontmatter fields produce `/` keys; the body is emitted as +`/agent_system_prompt`. See the [Prompty spec](https://github.com/microsoft/prompty) +for the file format. **Non-`.prompty` files in scope cause Init to fail.** + +``` +--- +name: Weather Agent +agent_role: Weather expert +agent_goal: Help users plan trips +--- +You are a friendly weather assistant. +``` + +produces: + +``` +weather/name = "Weather Agent" +weather/agent_role = "Weather expert" +weather/agent_goal = "Help users plan trips" +weather/agent_system_prompt = "You are a friendly weather assistant." +``` + +## Authentication + +The active auth profile is inferred from which metadata fields you set: + +1. `appId` → **GitHub App** profile. +2. `remoteUrl` begins with `git@` or `ssh://` → **SSH** profile. +3. `token` → **PAT** profile. +4. Otherwise → no auth (public HTTPS or local `file://`). + +Sensitive fields should be sourced from a configured secret store via +`secretKeyRef`. + +### PAT (HTTPS basic auth) + +Works with both GitHub classic PATs (`ghp_…`) and fine-grained PATs +(`github_pat_…`). + +```yaml +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "https://github.com/example/private-config.git" + - name: token + secretKeyRef: + name: github-pat + key: token +auth: + secretStore: +``` + +### SSH + +```yaml +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "git@github.com:example/private-config.git" + - name: privateKey + secretKeyRef: + name: git-ssh-deploy-key + key: privateKey + - name: knownHosts + secretKeyRef: + name: git-ssh-known-hosts + key: knownHosts +auth: + secretStore: +``` + +`insecureIgnoreHostKey: true` disables host-key verification — only use for +local development. A loud warning is logged at startup when this is enabled. + +### GitHub App + +```yaml +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "https://github.com/example/private-config.git" + - name: appId + value: "123456" + - name: installationId + value: "78901234" + - name: privateKey + secretKeyRef: + name: github-app-key + key: privateKey +auth: + secretStore: +``` + +Accepts both PKCS#1 and PKCS#8 PEM-encoded RSA keys. The component mints an +RS256 JWT, exchanges it for a 1-hour installation token, and refreshes the +token `refreshSkew` (default 5m) before expiry. + +## Behavioural notes + +- **`Get` is cache-only.** Returns the most-recently polled snapshot and may + be up to `pollInterval` old. Does not contact the remote. +- **`Subscribe` replays current state by default.** When `emitInitialState` + is true (the default), the handler receives the current snapshot + synchronously before `Subscribe` returns. +- **Returned items are deep copies** — callers own them and may mutate + freely without affecting the store. +- **Deletions are signalled** via `Item{Value: "", Metadata: {"deleted": "true"}}`, + same shape as the kubernetes ConfigMap configuration store. +- **Read-only**: the component never writes to the upstream. Configuration + changes must be made by committing through your normal git workflow (PR + review, branch protection, etc.). +- **Rate-limit handling**: on HTTP 429 from the GitHub API (used by the + GitHub App installation-token exchange), or a transport-level rate-limit + error from go-git, the poll loop pauses for `rateLimitRetryAfter` + (default 5m) — or the server-supplied `Retry-After` when present — + before the next tick. +- **`.git/` is always excluded** from the worktree walk regardless of + `includeHidden`, so credentials in `.git/config` cannot leak. A `path` + containing the `.git` segment is rejected at Init. +- **Per-file size cap.** Files larger than `maxFileSize` (default 1 MiB) + are skipped with a warning. +- **LRU snapshot cache.** Per-subscriber diffs reuse cached snapshots keyed + by commit SHA (default size 4). On a miss the diff over-emits a single + batch — idempotent on the receiver. + +## Limitations + +- **Single GitHub App installation per component.** Multi-tenant routing + (different repos via different installations on the same component) is + not supported. + +## Daprd registration + +This component requires a paired `dapr/dapr` PR registering +`configuration.git` so that daprd can load it via component spec. See +`cmd/daprd/components/configuration_redis.go` for the registration shape. +Until that PR lands, the component can be exercised via the certification +test in `tests/certification/configuration/git/`. diff --git a/configuration/git/auth.go b/configuration/git/auth.go new file mode 100644 index 0000000000..1307dcb10d --- /dev/null +++ b/configuration/git/auth.go @@ -0,0 +1,72 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "fmt" + + "github.com/dapr/components-contrib/configuration/git/auth" + "github.com/dapr/kit/logger" +) + +// selectAuth resolves the active auth strategy from metadata and builds it +// via the auth sub-package. log is guaranteed non-nil by callers (it +// threads through from the Store's constructor which always supplies a +// logger). +func selectAuth(m *metadata, log logger.Logger) (auth.Strategy, error) { + switch m.resolveAuthMode() { + case authModeNone: + return auth.NewNone(), nil + case authModePAT: + username := "" + if m.Username != nil { + username = *m.Username + } + return auth.NewPAT(username, m.Token), nil + case authModeSSH: + return auth.NewSSH(auth.SSHConfig{ + User: m.user(), + PrivateKey: m.PrivateKey, + PrivateKeyPath: ptrDeref(m.PrivateKeyPath), + Passphrase: m.Passphrase, + KnownHosts: ptrDeref(m.KnownHosts), + KnownHostsPath: ptrDeref(m.KnownHostsPath), + InsecureIgnoreHostKey: m.insecureIgnoreHostKey(), + }, log) + case authModeGithubApp: + return auth.NewGitHubApp(auth.GitHubAppConfig{ + AppID: derefInt64(m.AppID), + InstallationID: derefInt64(m.InstallationID), + PrivateKey: m.PrivateKey, + PrivateKeyPath: ptrDeref(m.PrivateKeyPath), + APIBase: m.apiBase(), + RefreshSkew: m.refreshSkew(), + }, log, auth.DefaultInstallationTokenFetcher) + } + return nil, fmt.Errorf("unsupported auth mode %q", m.resolveAuthMode()) +} + +func ptrDeref(p *string) string { + if p == nil { + return "" + } + return *p +} + +func derefInt64(p *int64) int64 { + if p == nil { + return 0 + } + return *p +} diff --git a/configuration/git/auth/auth_test.go b/configuration/git/auth/auth_test.go new file mode 100644 index 0000000000..217fc03b18 --- /dev/null +++ b/configuration/git/auth/auth_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + xssh "golang.org/x/crypto/ssh" + + "github.com/dapr/kit/logger" +) + +func signerFromPrivateKey(priv *rsa.PrivateKey) (xssh.Signer, error) { + return xssh.NewSignerFromKey(priv) +} + +func base64Encode(b []byte) string { return base64.StdEncoding.EncodeToString(b) } + +// generateTestRSAPEM produces a fresh PEM-encoded RSA private key. Used to +// keep the test self-contained and avoid committing test keys. +func generateTestRSAPEM(t *testing.T) string { + t.Helper() + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der := x509.MarshalPKCS1PrivateKey(priv) + block := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: der} + return string(pem.EncodeToMemory(block)) +} + +// generateTestSSHPrivateKey returns a PEM-encoded private key suitable for +// gitssh.NewPublicKeys. +func generateTestSSHPrivateKey(t *testing.T) string { + t.Helper() + return generateTestRSAPEM(t) +} + +func TestGitHubAppAuth_RefreshCadence(t *testing.T) { + keyPEM := generateTestRSAPEM(t) + skew := 5 * time.Minute + + var calls atomic.Int32 + expiry := time.Now().Add(1 * time.Hour) + fetcher := func(_ context.Context, _ *http.Client, _ string, _ int64, _ string) (*InstallationToken, error) { + calls.Add(1) + return &InstallationToken{Token: "ghs_abc", ExpiresAt: expiry}, nil + } + + s, err := NewGitHubApp(GitHubAppConfig{ + AppID: 123, + InstallationID: 456, + PrivateKey: keyPEM, + RefreshSkew: skew, + }, logger.NewLogger("test"), fetcher) + require.NoError(t, err) + a := s.(*githubAppStrategy) + + // First call mints JWT and fetches. + for range 5 { + am, methodErr := a.AuthMethod(t.Context()) + require.NoError(t, methodErr) + ba := am.(*githttp.BasicAuth) + assert.Equal(t, "x-access-token", ba.Username) + assert.Equal(t, "ghs_abc", ba.Password) + } + assert.Equal(t, int32(1), calls.Load(), "should reuse cached token until skew window") + + // Force expiry within skew → should refresh. + a.mu.Lock() + a.cached.ExpiresAt = time.Now().Add(skew - time.Minute) + a.mu.Unlock() + + _, err = a.AuthMethod(t.Context()) + require.NoError(t, err) + assert.Equal(t, int32(2), calls.Load(), "should refresh once expiry is within skew") +} + +func TestSSHAuth_LoadKeyFromPath(t *testing.T) { + dir := t.TempDir() + keyPath := dir + "/key" + require.NoError(t, os.WriteFile(keyPath, []byte(generateTestSSHPrivateKey(t)), 0o600)) + + s, err := NewSSH(SSHConfig{ + PrivateKeyPath: keyPath, + InsecureIgnoreHostKey: true, + }, logger.NewLogger("test")) + require.NoError(t, err) + require.IsType(t, &sshStrategy{}, s) +} + +func TestSSHAuth_KnownHostsFromInlineString(t *testing.T) { + // Build a known_hosts entry using a freshly generated key and verify + // the SSH auth strategy successfully constructs a HostKeyCallback from + // it. + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + signer, err := signerFromPrivateKey(priv) + require.NoError(t, err) + known := "github.com " + signer.PublicKey().Type() + " " + base64Encode(signer.PublicKey().Marshal()) + "\n" + + s, err := NewSSH(SSHConfig{ + PrivateKey: generateTestRSAPEM(t), + KnownHosts: known, + }, logger.NewLogger("test")) + require.NoError(t, err) + require.IsType(t, &sshStrategy{}, s) +} + +func TestGitHubAppAuth_LoadKeyFromPath(t *testing.T) { + dir := t.TempDir() + keyPath := dir + "/app.pem" + require.NoError(t, os.WriteFile(keyPath, []byte(generateTestRSAPEM(t)), 0o600)) + + s, err := NewGitHubApp(GitHubAppConfig{ + AppID: 1, + InstallationID: 2, + PrivateKeyPath: keyPath, + }, logger.NewLogger("test"), func(_ context.Context, _ *http.Client, _ string, _ int64, _ string) (*InstallationToken, error) { + return &InstallationToken{Token: "tok", ExpiresAt: time.Now().Add(time.Hour)}, nil + }) + require.NoError(t, err) + _, err = s.AuthMethod(t.Context()) + require.NoError(t, err) +} + +// TestInstallationTokenFetcher_RateLimitRetry exercises the 429 / +// Retry-After path: the first response 429s with a short Retry-After; the +// second succeeds. The fetcher must wait, retry, and return the success +// token. +func TestInstallationTokenFetcher_RateLimitRetry(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + n := hits.Add(1) + if n == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"token":"ghs_recovered","expires_at":"2030-01-01T00:00:00Z"}`)) + })) + t.Cleanup(srv.Close) + + tok, err := DefaultInstallationTokenFetcher(t.Context(), srv.Client(), srv.URL, 42, "jwt-stub") + require.NoError(t, err) + require.NotNil(t, tok) + assert.Equal(t, "ghs_recovered", tok.Token) + assert.Equal(t, int32(2), hits.Load(), "fetcher must retry exactly once after 429") +} + +// TestInstallationTokenFetcher_RateLimitTwice asserts that two consecutive +// 429s surface as a *RateLimitError to the caller (used by the poll loop +// to trigger back-off). +func TestInstallationTokenFetcher_RateLimitTwice(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(srv.Close) + + _, err := DefaultInstallationTokenFetcher(t.Context(), srv.Client(), srv.URL, 42, "jwt-stub") + require.Error(t, err) + var rl *RateLimitError + assert.ErrorAs(t, err, &rl, "second 429 must produce a *RateLimitError") +} + +func TestParseRetryAfter(t *testing.T) { + t.Run("empty", func(t *testing.T) { + assert.Equal(t, time.Duration(0), parseRetryAfter("", time.Now())) + }) + t.Run("zero", func(t *testing.T) { + assert.Equal(t, time.Duration(0), parseRetryAfter("0", time.Now())) + }) + t.Run("seconds", func(t *testing.T) { + assert.Equal(t, 30*time.Second, parseRetryAfter("30", time.Now())) + }) + t.Run("http date in future", func(t *testing.T) { + future := time.Now().Add(2 * time.Minute).UTC() + got := parseRetryAfter(future.Format(http.TimeFormat), time.Now()) + // HTTP-date is second-resolution; allow ±2s for the round-trip + // through http.ParseTime + the wall-clock advance between + // formatting and parsing. + diff := got - 2*time.Minute + if diff < 0 { + diff = -diff + } + assert.LessOrEqual(t, diff, 2*time.Second) + }) + t.Run("http date in past", func(t *testing.T) { + past := time.Now().Add(-2 * time.Minute).UTC() + assert.Equal(t, time.Duration(0), parseRetryAfter(past.Format(http.TimeFormat), time.Now())) + }) + t.Run("unparseable", func(t *testing.T) { + assert.Equal(t, time.Duration(0), parseRetryAfter("soon", time.Now())) + }) +} + +// TestIsRateLimited covers the 403-with-rate-limit-headers case used for +// GitHub secondary rate limits. +func TestIsRateLimited(t *testing.T) { + cases := []struct { + name string + code int + hdr http.Header + want bool + }{ + {"429", http.StatusTooManyRequests, http.Header{}, true}, + {"403 with remaining 0", http.StatusForbidden, http.Header{"X-Ratelimit-Remaining": {"0"}}, true}, + {"403 with retry-after", http.StatusForbidden, http.Header{"Retry-After": {"60"}}, true}, + {"403 without rate-limit headers", http.StatusForbidden, http.Header{}, false}, + {"200", http.StatusOK, http.Header{}, false}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + r := &http.Response{StatusCode: tt.code, Header: tt.hdr} + assert.Equal(t, tt.want, isRateLimited(r)) + }) + } +} + +// TestIsTransportRateLimit covers the go-git transport-error pattern check. +func TestIsTransportRateLimit(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"unrelated", errors.New("connection reset"), false}, + {"429 in message", errors.New("unexpected status 429"), true}, + {"rate limit text", errors.New("API rate limit exceeded"), true}, + {"secondary rate limit", errors.New("you have exceeded a secondary rate limit"), true}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsTransportRateLimit(tt.err)) + }) + } +} diff --git a/configuration/git/auth/githubapp.go b/configuration/git/auth/githubapp.go new file mode 100644 index 0000000000..6cb1623688 --- /dev/null +++ b/configuration/git/auth/githubapp.go @@ -0,0 +1,152 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net/http" + "strconv" + "sync" + "time" + + "github.com/go-git/go-git/v5/plumbing/transport" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwt" + + "github.com/dapr/kit/logger" +) + +// GitHubAppConfig collects the parameters required to construct a GitHub +// App auth strategy. Exactly one of PrivateKey or PrivateKeyPath must be +// non-empty. +type GitHubAppConfig struct { + AppID int64 + InstallationID int64 + PrivateKey string // Inline PEM-encoded RSA private key. + PrivateKeyPath string // Path to a PEM-encoded RSA private key on disk. + APIBase string // GitHub API base URL (default "https://api.github.com"). + RefreshSkew time.Duration // Token refreshes when remaining lifetime is less than this. +} + +type githubAppStrategy struct { + appID int64 + installationID int64 + apiBase string + privateKey *rsa.PrivateKey + skew time.Duration + fetcher InstallationTokenFetcher + httpClient *http.Client + log logger.Logger + + mu sync.Mutex + cached *InstallationToken +} + +// NewGitHubApp returns a GitHub App auth Strategy. log must be non-nil. +// fetcher is the installation-token exchange function; pass +// DefaultInstallationTokenFetcher unless overriding for tests. +func NewGitHubApp(cfg GitHubAppConfig, log logger.Logger, fetcher InstallationTokenFetcher) (Strategy, error) { + keyBytes, err := loadKeyMaterial("githubApp", cfg.PrivateKey, cfg.PrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("githubApp: %w", err) + } + priv, err := parseRSAPrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("githubApp: parse private key: %w", err) + } + apiBase := cfg.APIBase + if apiBase == "" { + apiBase = "https://api.github.com" + } + return &githubAppStrategy{ + appID: cfg.AppID, + installationID: cfg.InstallationID, + apiBase: apiBase, + privateKey: priv, + skew: cfg.RefreshSkew, + fetcher: fetcher, + httpClient: &http.Client{Timeout: 10 * time.Second}, + log: log, + }, nil +} + +func (a *githubAppStrategy) AuthMethod(ctx context.Context) (transport.AuthMethod, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.cached != nil && time.Until(a.cached.ExpiresAt) > a.skew { + return basicAuthFromInstallation(a.cached.Token), nil + } + jwtToken, err := a.mintJWT() + if err != nil { + return nil, fmt.Errorf("githubApp: mint JWT: %w", err) + } + tok, err := a.fetcher(ctx, a.httpClient, a.apiBase, a.installationID, jwtToken) + if err != nil { + return nil, fmt.Errorf("githubApp: exchange installation token: %w", err) + } + a.cached = tok + return basicAuthFromInstallation(tok.Token), nil +} + +func (a *githubAppStrategy) Close() error { return nil } + +func (a *githubAppStrategy) mintJWT() (string, error) { + now := time.Now() + tok, err := jwt.NewBuilder(). + Issuer(strconv.FormatInt(a.appID, 10)). + IssuedAt(now.Add(-30 * time.Second)). + Expiration(now.Add(10 * time.Minute)). + Build() + if err != nil { + return "", fmt.Errorf("build jwt: %w", err) + } + signed, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, a.privateKey)) + if err != nil { + return "", fmt.Errorf("sign jwt: %w", err) + } + return string(signed), nil +} + +// parseRSAPrivateKey decodes a PEM-encoded RSA private key. Accepts both +// PKCS#1 ("RSA PRIVATE KEY") and PKCS#8 ("PRIVATE KEY") encodings — GitHub +// Apps can be downloaded in either form. +func parseRSAPrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil, errors.New("no PEM block found") + } + if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { + return key, nil + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse private key: %w", err) + } + rsaKey, ok := parsed.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("expected RSA private key, got %T", parsed) + } + return rsaKey, nil +} + +func basicAuthFromInstallation(token string) *githttp.BasicAuth { + return &githttp.BasicAuth{Username: "x-access-token", Password: token} +} diff --git a/configuration/git/auth/installation.go b/configuration/git/auth/installation.go new file mode 100644 index 0000000000..6118ba6c5e --- /dev/null +++ b/configuration/git/auth/installation.go @@ -0,0 +1,165 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "time" +) + +// InstallationToken is the GitHub-App installation access token returned +// by the GitHub API. +type InstallationToken struct { + Token string + ExpiresAt time.Time +} + +// InstallationTokenFetcher exchanges a JWT for a GitHub App installation +// token. It is injectable so tests can avoid real HTTP. The client argument +// is supplied by the caller so connection pooling is reused across +// refreshes; tests pass a stub or a httptest-backed client. +type InstallationTokenFetcher func(ctx context.Context, client *http.Client, apiBase string, installationID int64, jwtToken string) (*InstallationToken, error) + +// RateLimitError indicates the GitHub API rejected the request with a 429 +// or 403-with-rate-limit-headers response. RetryAfter conveys the duration +// the server asked us to wait (zero if no Retry-After header was present, +// in which case the caller should fall back to its configured retry-after +// default). +type RateLimitError struct { + RetryAfter time.Duration + Err error +} + +func (e *RateLimitError) Error() string { + if e.RetryAfter > 0 { + return fmt.Sprintf("github api rate-limited (retry-after %s): %v", e.RetryAfter, e.Err) + } + return fmt.Sprintf("github api rate-limited: %v", e.Err) +} + +func (e *RateLimitError) Unwrap() error { return e.Err } + +// DefaultInstallationTokenFetcher exchanges a JWT for an installation +// token against the real GitHub API. +// +// On a 429 response (or 403 with rate-limit headers — GitHub uses 403 for +// secondary rate limits), Retry-After is parsed and the call is retried +// once. If the second attempt also fails with a rate-limit response, a +// *RateLimitError is returned so the poll loop can back off. +// +// Error messages exclude the response body — a misconfigured or +// compromised apiBase could echo sensitive content back. The 64 KiB +// LimitReader caps memory in case the body is unexpectedly large. +func DefaultInstallationTokenFetcher(ctx context.Context, client *http.Client, apiBase string, installationID int64, jwtToken string) (*InstallationToken, error) { + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + tok, err := doInstallationTokenRequest(ctx, client, apiBase, installationID, jwtToken) + var rateErr *RateLimitError + if errors.As(err, &rateErr) && rateErr.RetryAfter > 0 { + // Wait for the server-suggested duration, then retry once. + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(rateErr.RetryAfter): + } + tok, err = doInstallationTokenRequest(ctx, client, apiBase, installationID, jwtToken) + } + return tok, err +} + +func doInstallationTokenRequest(ctx context.Context, client *http.Client, apiBase string, installationID int64, jwtToken string) (*InstallationToken, error) { + url := fmt.Sprintf("%s/app/installations/%d/access_tokens", apiBase, installationID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+jwtToken) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + + if isRateLimited(resp) { + return nil, &RateLimitError{ + RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()), + Err: fmt.Errorf("github api: status %d (installation %d)", resp.StatusCode, installationID), + } + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("github api: status %d (installation %d)", resp.StatusCode, installationID) + } + + var payload struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + if payload.Token == "" { + return nil, errors.New("github api returned empty token") + } + return &InstallationToken{Token: payload.Token, ExpiresAt: payload.ExpiresAt}, nil +} + +// isRateLimited returns true if the response indicates the request was +// rate-limited by GitHub. GitHub uses both 429 and 403 (with the +// X-RateLimit-Remaining: 0 header) for primary and secondary rate limits. +func isRateLimited(resp *http.Response) bool { + if resp.StatusCode == http.StatusTooManyRequests { + return true + } + if resp.StatusCode == http.StatusForbidden && + (resp.Header.Get("X-RateLimit-Remaining") == "0" || + resp.Header.Get("Retry-After") != "") { + return true + } + return false +} + +// parseRetryAfter interprets an HTTP Retry-After header value, which may +// be either an integer number of seconds or an HTTP-date. Returns 0 when +// the header is empty or unparseable. +func parseRetryAfter(v string, now time.Time) time.Duration { + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + if when, err := http.ParseTime(v); err == nil { + if d := time.Until(when); d > 0 { + return d + } + // HTTP-date already in the past — fall through to "no wait". + _ = now + } + return 0 +} diff --git a/configuration/git/auth/keymaterial.go b/configuration/git/auth/keymaterial.go new file mode 100644 index 0000000000..528d3886e6 --- /dev/null +++ b/configuration/git/auth/keymaterial.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "os" +) + +// loadKeyMaterial returns inline PEM bytes if non-empty, else reads them +// from the supplied path. Returns an error if neither is provided. +func loadKeyMaterial(kind, inline, path string) ([]byte, error) { + if inline != "" { + return []byte(inline), nil + } + if path != "" { + return os.ReadFile(path) + } + return nil, fmt.Errorf("%s: inline key or path is required", kind) +} diff --git a/configuration/git/auth/none.go b/configuration/git/auth/none.go new file mode 100644 index 0000000000..6e875c1278 --- /dev/null +++ b/configuration/git/auth/none.go @@ -0,0 +1,29 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + + "github.com/go-git/go-git/v5/plumbing/transport" +) + +type none struct{} + +// NewNone returns a Strategy that performs no authentication — used for +// public HTTPS repos and file:// URLs. +func NewNone() Strategy { return &none{} } + +func (*none) AuthMethod(context.Context) (transport.AuthMethod, error) { return nil, nil } +func (*none) Close() error { return nil } diff --git a/configuration/git/auth/pat.go b/configuration/git/auth/pat.go new file mode 100644 index 0000000000..d35bff03f8 --- /dev/null +++ b/configuration/git/auth/pat.go @@ -0,0 +1,42 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + + "github.com/go-git/go-git/v5/plumbing/transport" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +type pat struct { + username string + token string +} + +// NewPAT returns a Strategy for HTTPS basic auth using a personal access +// token. An empty username is replaced with the GitHub-recommended +// "x-access-token". +func NewPAT(username, token string) Strategy { + if username == "" { + username = "x-access-token" + } + return &pat{username: username, token: token} +} + +func (a *pat) AuthMethod(context.Context) (transport.AuthMethod, error) { + return &githttp.BasicAuth{Username: a.username, Password: a.token}, nil +} + +func (a *pat) Close() error { return nil } diff --git a/configuration/git/auth/ssh.go b/configuration/git/auth/ssh.go new file mode 100644 index 0000000000..a8cb4dceb1 --- /dev/null +++ b/configuration/git/auth/ssh.go @@ -0,0 +1,119 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/go-git/go-git/v5/plumbing/transport" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" + + "github.com/dapr/kit/logger" +) + +// SSHConfig collects the parameters required to construct an SSH auth +// strategy. Exactly one of PrivateKey or PrivateKeyPath must be non-empty. +// Exactly one of KnownHosts or KnownHostsPath must be non-empty unless +// InsecureIgnoreHostKey is true. +type SSHConfig struct { + User string // SSH user; defaults to "git" if empty. + PrivateKey string // Inline PEM-encoded SSH private key. + PrivateKeyPath string // Path to a PEM-encoded SSH private key on disk. + Passphrase string // Passphrase for the SSH private key. + KnownHosts string // Inline OpenSSH known_hosts entries. + KnownHostsPath string // Path to an OpenSSH known_hosts file. + InsecureIgnoreHostKey bool // When true, disables host-key verification. +} + +type sshStrategy struct { + user string + keys *gitssh.PublicKeys + knownHosts ssh.HostKeyCallback + insecure bool +} + +// NewSSH returns an SSH auth Strategy. When cfg.InsecureIgnoreHostKey is +// true a warning is logged and host-key verification is disabled — never +// safe in production. log must be non-nil. +func NewSSH(cfg SSHConfig, log logger.Logger) (Strategy, error) { + user := cfg.User + if user == "" { + user = "git" + } + keyBytes, err := loadKeyMaterial("ssh", cfg.PrivateKey, cfg.PrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("ssh: %w", err) + } + pk, err := gitssh.NewPublicKeys(user, keyBytes, cfg.Passphrase) + if err != nil { + return nil, fmt.Errorf("ssh: parse private key: %w", err) + } + a := &sshStrategy{user: user, keys: pk} + + if cfg.InsecureIgnoreHostKey { + log.Warnf("git ssh: insecureIgnoreHostKey=true — host key verification is DISABLED. Do not use in production.") + a.insecure = true + // Explicitly opting in to disabled host-key verification because the + // operator set insecureIgnoreHostKey=true. Loud-logged above. + pk.HostKeyCallback = ssh.InsecureIgnoreHostKey() //nolint:gosec // G106: documented opt-in via metadata + return a, nil + } + + cb, err := buildKnownHostsCallback(cfg.KnownHosts, cfg.KnownHostsPath) + if err != nil { + return nil, fmt.Errorf("ssh: known_hosts: %w", err) + } + pk.HostKeyCallback = cb + a.knownHosts = cb + return a, nil +} + +func (a *sshStrategy) AuthMethod(context.Context) (transport.AuthMethod, error) { + return a.keys, nil +} + +func (a *sshStrategy) Close() error { return nil } + +func buildKnownHostsCallback(inline, path string) (ssh.HostKeyCallback, error) { + if path != "" { + return knownhosts.New(path) + } + if inline != "" { + // `knownhosts.New` requires a file path. Persist the inline data to a + // temp file just long enough to hand it off; the file is removed + // before we return regardless of outcome. + f, err := os.CreateTemp("", "dapr-knownhosts-*") + if err != nil { + return nil, fmt.Errorf("create knownHosts temp file: %w", err) + } + tmp := f.Name() + defer func() { + _ = f.Close() + _ = os.Remove(tmp) + }() + if _, err := f.WriteString(inline); err != nil { + return nil, fmt.Errorf("write knownHosts: %w", err) + } + if err := f.Close(); err != nil { + return nil, fmt.Errorf("close knownHosts: %w", err) + } + return knownhosts.New(tmp) + } + return nil, errors.New("no knownHosts/knownHostsPath configured and insecureIgnoreHostKey is false") +} diff --git a/configuration/git/auth/strategy.go b/configuration/git/auth/strategy.go new file mode 100644 index 0000000000..f695b47cf2 --- /dev/null +++ b/configuration/git/auth/strategy.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package auth implements the auth strategies used by configuration.git. +// Each strategy resolves a go-git AuthMethod, refreshing internal credentials +// over time if needed (GitHub App installation tokens have a ~1h TTL). +package auth + +import ( + "context" + "strings" + + "github.com/go-git/go-git/v5/plumbing/transport" +) + +// Strategy resolves a go-git AuthMethod for use against the upstream git +// remote. Implementations must be safe for concurrent calls to AuthMethod +// from the poller and clone paths. +type Strategy interface { + AuthMethod(ctx context.Context) (transport.AuthMethod, error) + Close() error +} + +// IsTransportRateLimit returns true if err appears to be a rate-limit +// response surfaced from a go-git HTTP/HTTPS transport. go-git does not +// expose structured response status, so this is a string-pattern check — +// best-effort but it covers GitHub's typical error shapes. +func IsTransportRateLimit(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "429") || + strings.Contains(msg, "rate limit") || + strings.Contains(msg, "secondary rate limit") +} diff --git a/configuration/git/auth_test.go b/configuration/git/auth_test.go new file mode 100644 index 0000000000..31dbd57107 --- /dev/null +++ b/configuration/git/auth_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + + "github.com/dapr/kit/logger" +) + +// TestSelectAuth verifies that selectAuth (which wires metadata into the +// auth sub-package) picks the correct strategy and produces the expected +// AuthMethod. Tests for the strategies themselves live in +// configuration/git/auth. +func TestSelectAuth(t *testing.T) { + t.Run("none", func(t *testing.T) { + m := &metadata{RemoteURL: "https://example.com/repo.git"} + s, err := selectAuth(m, logger.NewLogger("test")) + require.NoError(t, err) + am, err := s.AuthMethod(t.Context()) + require.NoError(t, err) + assert.Nil(t, am) + }) + + t.Run("pat classic token", func(t *testing.T) { + m := &metadata{RemoteURL: "https://example.com/repo.git", Token: "ghp_classic1234567890abcdef"} + s, err := selectAuth(m, logger.NewLogger("test")) + require.NoError(t, err) + am, err := s.AuthMethod(t.Context()) + require.NoError(t, err) + ba, ok := am.(*githttp.BasicAuth) + require.True(t, ok) + assert.Equal(t, "x-access-token", ba.Username) + assert.Equal(t, "ghp_classic1234567890abcdef", ba.Password) + }) + + t.Run("pat fine-grained token", func(t *testing.T) { + // Fine-grained PATs use the `github_pat_` prefix. They go through + // the same HTTP Basic Auth path and must produce an identically + // shaped AuthMethod. + m := &metadata{ + RemoteURL: "https://example.com/repo.git", + Token: "github_pat_finegrained_abcdef1234567890", + } + s, err := selectAuth(m, logger.NewLogger("test")) + require.NoError(t, err) + am, err := s.AuthMethod(t.Context()) + require.NoError(t, err) + ba, ok := am.(*githttp.BasicAuth) + require.True(t, ok) + assert.Equal(t, "x-access-token", ba.Username) + assert.Equal(t, "github_pat_finegrained_abcdef1234567890", ba.Password) + }) + + t.Run("pat with explicit username", func(t *testing.T) { + user := "alice" + m := &metadata{RemoteURL: "https://example.com/repo.git", Username: &user, Token: "ghp_abc"} + s, err := selectAuth(m, logger.NewLogger("test")) + require.NoError(t, err) + am, err := s.AuthMethod(t.Context()) + require.NoError(t, err) + ba := am.(*githttp.BasicAuth) + assert.Equal(t, "alice", ba.Username) + }) + + t.Run("auto-detect ssh from url", func(t *testing.T) { + key := generateTestSSHPrivateKey(t) + insecure := true + m := &metadata{RemoteURL: "git@example.com:org/repo.git", PrivateKey: key, InsecureIgnoreHostKey: &insecure} + _, err := selectAuth(m, logger.NewLogger("test")) + require.NoError(t, err) + }) + + t.Run("ssh missing known_hosts when not insecure", func(t *testing.T) { + key := generateTestSSHPrivateKey(t) + m := &metadata{RemoteURL: "git@example.com:org/repo.git", PrivateKey: key} + _, err := selectAuth(m, logger.NewLogger("test")) + require.Error(t, err) + assert.Contains(t, err.Error(), "knownHosts") + }) +} + +// generateTestSSHPrivateKey returns a PEM-encoded RSA private key suitable +// for the SSH auth strategy in tests. Used here to keep TestSelectAuth +// self-contained. +func generateTestSSHPrivateKey(t *testing.T) string { + t.Helper() + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der := x509.MarshalPKCS1PrivateKey(priv) + block := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: der} + return string(pem.EncodeToMemory(block)) +} diff --git a/configuration/git/clone.go b/configuration/git/clone.go new file mode 100644 index 0000000000..69bc1c7d52 --- /dev/null +++ b/configuration/git/clone.go @@ -0,0 +1,96 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "errors" + "fmt" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + + "github.com/dapr/components-contrib/configuration/git/auth" +) + +const remoteName = "origin" + +// cloneFresh clones the repo at url into dir. The clone is bound to the given +// branch and tracks `origin`. Auth is fetched per-call so GitHub App tokens +// rotate naturally. +func cloneFresh(ctx context.Context, dir, url, branch string, depth int, strategy auth.Strategy) (*gogit.Repository, error) { + method, err := strategy.AuthMethod(ctx) + if err != nil { + return nil, fmt.Errorf("auth: %w", err) + } + opts := &gogit.CloneOptions{ + URL: url, + Auth: method, + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: true, + Depth: depth, + } + repo, err := gogit.PlainCloneContext(ctx, dir, false, opts) + if err != nil { + return nil, fmt.Errorf("clone %s: %w", url, err) + } + return repo, nil +} + +// fetchIfChanged fetches the configured branch and, if the upstream tip has +// moved, resets the worktree to the new tip. Returns (head, changed, err) +// where `changed` is false when the remote ref is unchanged. Uses go-git's +// own NoErrAlreadyUpToDate signal rather than a separate ls-remote probe; +// this avoids downloading the full ref advertisement on every poll tick. +func fetchIfChanged(ctx context.Context, repo *gogit.Repository, branch string, strategy auth.Strategy) (plumbing.Hash, bool, error) { + method, err := strategy.AuthMethod(ctx) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("auth: %w", err) + } + branchRef := plumbing.NewBranchReferenceName(branch) + remoteRef := plumbing.NewRemoteReferenceName(remoteName, branch) + + fetchSpec := config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)) + fetchErr := repo.FetchContext(ctx, &gogit.FetchOptions{ + RemoteName: remoteName, + Auth: method, + RefSpecs: []config.RefSpec{fetchSpec}, + Force: true, + }) + upToDate := errors.Is(fetchErr, gogit.NoErrAlreadyUpToDate) + if fetchErr != nil && !upToDate { + return plumbing.ZeroHash, false, fmt.Errorf("fetch: %w", fetchErr) + } + + tip, err := repo.Reference(remoteRef, true) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("resolve remote ref: %w", err) + } + hash := tip.Hash() + + currentHead, err := repo.Head() + if err == nil && currentHead.Hash() == hash && upToDate { + return hash, false, nil + } + + wt, err := repo.Worktree() + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("worktree: %w", err) + } + if err := wt.Reset(&gogit.ResetOptions{Mode: gogit.HardReset, Commit: hash}); err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("hard reset to %s: %w", hash, err) + } + return hash, true, nil +} diff --git a/configuration/git/git.go b/configuration/git/git.go new file mode 100644 index 0000000000..3758d31f47 --- /dev/null +++ b/configuration/git/git.go @@ -0,0 +1,304 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package git implements a Dapr configuration store backed by a git +// repository. Configuration values live as files in a remote repository; +// the store clones the repo on Init, polls for new commits, and notifies +// subscribers when keys change. Three mapping modes are supported: +// +// - file (default): each file maps to one config item. +// - agentYaml: each YAML/JSON file is parsed as a flat map. +// - prompty: each .prompty file's frontmatter and body become keys. +// +// See metadata.yaml for the full configuration surface. +package git + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "sync" + "sync/atomic" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/google/uuid" + lru "github.com/hashicorp/golang-lru/v2" + + "github.com/dapr/components-contrib/configuration" + "github.com/dapr/components-contrib/configuration/git/auth" + contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/kit/logger" +) + +// pollIntervalWarnThreshold is the soft minimum below which we log a warning +// at startup. The hard floor is enforced in metadata.parse. +const pollIntervalWarnThreshold = 5 * time.Second + +var _ configuration.Store = (*ConfigurationStore)(nil) + +// ConfigurationStore is a git-backed Dapr configuration store. A single +// polling goroutine, started in Init, fans out to per-subscriber handlers. +type ConfigurationStore struct { + logger logger.Logger + + metadata metadata + auth auth.Strategy + mapper mappingStrategy + + repo *git.Repository + workdir string + + mu sync.Mutex + snapshot map[string]*configuration.Item + snapshotHead plumbing.Hash + subscriptions map[string]*subscription + // lru caches past snapshots keyed by commit SHA, used as diff bases when + // computing per-subscriber update events in refreshAndFanOut. Size is + // bounded by metadata.snapshotCacheSize (default 4). On a miss the diff + // falls back to a full re-emit of the intersection (idempotent on the + // receiver). + lru *lru.Cache[plumbing.Hash, map[string]*configuration.Item] + closed atomic.Bool + + pollerCancel context.CancelFunc + pollerWg sync.WaitGroup +} + +// NewGitConfigurationStore returns a new git-backed configuration store. +func NewGitConfigurationStore(log logger.Logger) configuration.Store { + return &ConfigurationStore{ + logger: log, + subscriptions: make(map[string]*subscription), + snapshot: map[string]*configuration.Item{}, + } +} + +// Init clones the upstream repository, builds an initial snapshot, and starts +// the polling goroutine. The poller is owned by the store and runs until +// Close is called; the Init ctx is used only for the initial clone and +// snapshot operations and does not bind the poller's lifetime. +// +// On any error after the workdir is created, this function cleans up the +// workdir and any partially-initialised auth strategy, so the Dapr runtime +// (which does not call Close on a failed Init) is not left with a leaked +// temp directory. +func (s *ConfigurationStore) Init(ctx context.Context, meta configuration.Metadata) error { + if err := s.metadata.parse(meta); err != nil { + return err + } + + strategy, err := selectAuth(&s.metadata, s.logger) + if err != nil { + return err + } + s.auth = strategy + + mapper, err := selectMapper(s.metadata.mappingMode()) + if err != nil { + return err + } + s.mapper = mapper + + dir, err := os.MkdirTemp("", "dapr-config-git-") + if err != nil { + return fmt.Errorf("create workdir: %w", err) + } + s.workdir = dir + + cleanupOnErr := true + defer func() { + if !cleanupOnErr { + return + } + if err := os.RemoveAll(dir); err != nil { + s.logger.Warnf("git config: failed to remove workdir %s during Init cleanup: %v", dir, err) + } + s.workdir = "" + if s.auth != nil { + if err := s.auth.Close(); err != nil { + s.logger.Warnf("git config: failed to close auth strategy during Init cleanup: %v", err) + } + } + }() + + cloneCtx, cancel := context.WithTimeout(ctx, s.metadata.fetchTimeout()) + repo, err := cloneFresh(cloneCtx, dir, s.metadata.RemoteURL, s.metadata.branch(), s.metadata.depth(), s.auth) + cancel() + if err != nil { + return err + } + s.repo = repo + + cache, err := lru.New[plumbing.Hash, map[string]*configuration.Item](s.metadata.snapshotCacheSize()) + if err != nil { + return fmt.Errorf("init lru: %w", err) + } + s.lru = cache + + if err := s.bootstrapSnapshot(); err != nil { + return fmt.Errorf("initial snapshot: %w", err) + } + + if s.metadata.pollInterval() < pollIntervalWarnThreshold { + s.logger.Warnf("git config: pollInterval %s is below recommended minimum %s — polling may overwhelm the git remote", + s.metadata.pollInterval(), pollIntervalWarnThreshold) + } + + pollerCtx, pollerCancel := context.WithCancel(context.Background()) + s.pollerCancel = pollerCancel + s.pollerWg.Add(1) + go s.pollLoop(pollerCtx) + + cleanupOnErr = false + return nil +} + +// bootstrapSnapshot builds the initial snapshot from the already-cloned +// worktree. No network I/O is performed here. +func (s *ConfigurationStore) bootstrapSnapshot() error { + head, err := s.repo.Head() + if err != nil { + return fmt.Errorf("resolve HEAD: %w", err) + } + hash := head.Hash() + entries, err := walkTree(s.workdir, s.metadata.path(), s.metadata.includeHidden(), s.metadata.maxFileSize(), s.logger) + if err != nil { + return err + } + snap, err := s.mapper.Map(entries, shortSHA(hash), s.logger) + if err != nil { + return err + } + s.mu.Lock() + s.snapshot = snap + s.snapshotHead = hash + s.lru.Add(hash, snap) + s.mu.Unlock() + return nil +} + +// Get returns the most-recently polled snapshot, filtered by req.Keys. +// It does not contact the upstream repository; the returned state may be up +// to pollInterval old. Use Subscribe to receive change notifications. +// +// Returned items are deep copies — callers own them and may mutate the +// values or metadata freely without affecting the store. +func (s *ConfigurationStore) Get(_ context.Context, req *configuration.GetRequest) (*configuration.GetResponse, error) { + if s.closed.Load() { + return nil, errors.New("configuration store is closed") + } + s.mu.Lock() + items := filterByKeys(s.snapshot, req.Keys) + s.mu.Unlock() + return &configuration.GetResponse{Items: items}, nil +} + +// Subscribe registers a handler for change notifications. When +// emitInitialState is true (the default), the current snapshot is delivered +// synchronously to the handler before this method returns, so callers do +// not need a separate Get + Subscribe pair. If the initial-state delivery +// fails, the subscription is rolled back and the error is returned. +// +// The handler receives a child context derived from ctx; cancelling the +// subscription via Unsubscribe also cancels that context. +func (s *ConfigurationStore) Subscribe(ctx context.Context, req *configuration.SubscribeRequest, handler configuration.UpdateHandler) (string, error) { + if s.closed.Load() { + return "", errors.New("configuration store is closed") + } + + id := uuid.NewString() + childCtx, cancel := context.WithCancel(ctx) + + s.mu.Lock() + sub := &subscription{ + id: id, + keys: append([]string(nil), req.Keys...), + handler: handler, + deliveredHEAD: s.snapshotHead, + ctx: childCtx, + cancel: cancel, + } + s.subscriptions[id] = sub + var initial map[string]*configuration.Item + if s.metadata.emitInitialState() { + initial = filterByKeys(s.snapshot, req.Keys) + } + s.mu.Unlock() + + if len(initial) > 0 { + if err := handler(childCtx, &configuration.UpdateEvent{ID: id, Items: initial}); err != nil { + s.mu.Lock() + delete(s.subscriptions, id) + s.mu.Unlock() + cancel() + return "", fmt.Errorf("initial state delivery failed: %w", err) + } + } + return id, nil +} + +// Unsubscribe removes a subscription. Errors if the id is unknown. +func (s *ConfigurationStore) Unsubscribe(_ context.Context, req *configuration.UnsubscribeRequest) error { + s.mu.Lock() + sub, ok := s.subscriptions[req.ID] + if !ok { + s.mu.Unlock() + return fmt.Errorf("subscription with id %s does not exist", req.ID) + } + delete(s.subscriptions, req.ID) + s.mu.Unlock() + sub.cancel() + return nil +} + +// Close stops the poller, cancels all subscriptions, and removes the workdir. +// Idempotent — subsequent calls return nil without performing any work. +func (s *ConfigurationStore) Close() error { + if !s.closed.CompareAndSwap(false, true) { + return nil + } + + s.mu.Lock() + for id, sub := range s.subscriptions { + sub.cancel() + delete(s.subscriptions, id) + } + s.mu.Unlock() + + if s.pollerCancel != nil { + s.pollerCancel() + s.pollerWg.Wait() + } + if s.auth != nil { + if err := s.auth.Close(); err != nil { + s.logger.Warnf("git config: auth close: %v", err) + } + } + if s.workdir != "" { + if err := os.RemoveAll(s.workdir); err != nil { + s.logger.Warnf("git config: workdir cleanup: %v", err) + } + } + return nil +} + +// GetComponentMetadata returns the schema for daprd auto-discovery. +func (s *ConfigurationStore) GetComponentMetadata() (info contribMetadata.MetadataMap) { + stub := metadata{} + contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(stub), &info, contribMetadata.ConfigurationStoreType) + return info +} diff --git a/configuration/git/git_test.go b/configuration/git/git_test.go new file mode 100644 index 0000000000..7504d50f4e --- /dev/null +++ b/configuration/git/git_test.go @@ -0,0 +1,545 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + + "github.com/dapr/components-contrib/configuration" + contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/kit/logger" +) + +// upstream is a lightweight fixture: a bare git repo on disk plus a private +// working clone. Tests commit through the working clone and the store under +// test reads from the bare repo over file://. +type upstream struct { + t *testing.T + dir string + bareDir string + workDir string + url string + branch string + repo *gogit.Repository + mu sync.Mutex + commitSeq int + signature *object.Signature + worktree *gogit.Worktree +} + +func newUpstream(t *testing.T) *upstream { + t.Helper() + // t.TempDir registers cleanup before any panic-prone setup runs, so a + // failed PlainInit/PlainClone never leaks the directory. + root := t.TempDir() + + bareDir := filepath.Join(root, "upstream.git") + workDir := filepath.Join(root, "work") + + _, err := gogit.PlainInit(bareDir, true) + require.NoError(t, err) + + repo, err := gogit.PlainClone(workDir, false, &gogit.CloneOptions{ + URL: bareDir, + }) + if err != nil { + // Empty bare repo has no refs to clone — initialise via PlainInit + // in the workdir, then add origin. + repo, err = gogit.PlainInit(workDir, false) + require.NoError(t, err) + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{bareDir}, + }) + require.NoError(t, err) + } + + wt, err := repo.Worktree() + require.NoError(t, err) + + u := &upstream{ + t: t, + dir: root, + bareDir: bareDir, + workDir: workDir, + url: fileURL(bareDir), + branch: "main", + repo: repo, + worktree: wt, + signature: &object.Signature{ + Name: "test", + Email: "test@example.com", + When: time.Unix(1700000000, 0), + }, + } + u.commit(map[string]string{}, "initial") + return u +} + +// fileURL produces a portable file:// URL for an absolute local path. +// On Unix the path already starts with `/`, so `file://` + path yields the +// canonical three-slash form. On Windows the path starts with a drive +// letter (e.g. `C:\foo`), so we normalise separators and prepend `/` +// to produce the standard `file:///C:/foo` form. +func fileURL(absPath string) string { + p := filepath.ToSlash(absPath) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return "file://" + p +} + +// commit writes the given files (relative paths under the workdir, value is +// content) and pushes a commit with the given message. Files not present in +// `files` are left as-is unless already deleted via removeFiles. +func (u *upstream) commit(files map[string]string, message string) { + u.mu.Lock() + defer u.mu.Unlock() + + for rel, content := range files { + p := filepath.Join(u.workDir, rel) + require.NoError(u.t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(u.t, os.WriteFile(p, []byte(content), 0o600)) + _, err := u.worktree.Add(rel) + require.NoError(u.t, err) + } + if u.commitSeq == 0 && len(files) == 0 { + // allow-empty initial commit + _, err := u.worktree.Commit(message, &gogit.CommitOptions{ + Author: u.signature, + Committer: u.signature, + AllowEmptyCommits: true, + }) + require.NoError(u.t, err) + } else { + _, err := u.worktree.Commit(message, &gogit.CommitOptions{ + Author: u.signature, + Committer: u.signature, + }) + require.NoError(u.t, err) + } + u.commitSeq++ + + require.NoError(u.t, u.repo.Push(&gogit.PushOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{config.RefSpec("refs/heads/master:refs/heads/main"), config.RefSpec("refs/heads/main:refs/heads/main")}, + })) +} + +// removeFiles deletes the given files in the working clone, stages, commits, +// and pushes. +func (u *upstream) removeFiles(rels []string, message string) { + u.mu.Lock() + defer u.mu.Unlock() + for _, rel := range rels { + p := filepath.Join(u.workDir, rel) + require.NoError(u.t, os.RemoveAll(p)) + _, err := u.worktree.Remove(rel) + require.NoError(u.t, err) + } + _, err := u.worktree.Commit(message, &gogit.CommitOptions{ + Author: u.signature, + Committer: u.signature, + }) + require.NoError(u.t, err) + require.NoError(u.t, u.repo.Push(&gogit.PushOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{config.RefSpec("refs/heads/master:refs/heads/main"), config.RefSpec("refs/heads/main:refs/heads/main")}, + })) +} + +func newTestStore(t *testing.T, u *upstream, props map[string]string) *ConfigurationStore { + t.Helper() + if props == nil { + props = map[string]string{} + } + props["remoteUrl"] = u.url + if _, ok := props["pollInterval"]; !ok { + props["pollInterval"] = "1s" + } + if props["mappingMode"] == "" { + props["mappingMode"] = "file" + } + + s := NewGitConfigurationStore(logger.NewLogger("test")).(*ConfigurationStore) + require.NoError(t, s.Init(t.Context(), configuration.Metadata{ + Base: contribMetadata.Base{Properties: props}, + })) + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestGitStore_InitAndGet(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{ + "agent_role.txt": "Weather expert", + "agent_goal.txt": "Help users", + }, "seed") + + s := newTestStore(t, u, nil) + resp, err := s.Get(t.Context(), &configuration.GetRequest{}) + require.NoError(t, err) + assert.Equal(t, "Weather expert", resp.Items["agent_role.txt"].Value) + assert.Equal(t, "Help users", resp.Items["agent_goal.txt"].Value) +} + +func TestGitStore_SubscribeReplaysInitialState(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{ + "agent_role.txt": "Weather expert", + }, "seed") + + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 4) + id, err := s.Subscribe(t.Context(), + &configuration.SubscribeRequest{}, + makeChanHandler(events), + ) + require.NoError(t, err) + require.NotEmpty(t, id) + + e := drainOne(t, events) + assert.Equal(t, "Weather expert", e.Items["agent_role.txt"].Value) +} + +func TestGitStore_HotReloadOnCommit(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"agent_role.txt": "Weather expert"}, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 16) + _, err := s.Subscribe(t.Context(), + &configuration.SubscribeRequest{}, + makeChanHandler(events), + ) + require.NoError(t, err) + drainOne(t, events) // initial state + + u.commit(map[string]string{"agent_role.txt": "Travel expert"}, "update role") + + require.Eventually(t, func() bool { + select { + case e := <-events: + if v, ok := e.Items["agent_role.txt"]; ok && v.Value == "Travel expert" { + return true + } + return false + default: + return false + } + }, 5*time.Second, 50*time.Millisecond, "expected role update event") +} + +func TestGitStore_DeletionEmitsDeletedMetadata(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{ + "a.txt": "alpha", + "b.txt": "beta", + }, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 16) + _, err := s.Subscribe(t.Context(), + &configuration.SubscribeRequest{}, + makeChanHandler(events), + ) + require.NoError(t, err) + drainOne(t, events) + + u.removeFiles([]string{"b.txt"}, "delete b") + + require.Eventually(t, func() bool { + select { + case e := <-events: + if v, ok := e.Items["b.txt"]; ok && v.Value == "" && v.Metadata["deleted"] == "true" { + return true + } + return false + default: + return false + } + }, 5*time.Second, 50*time.Millisecond, "expected deletion event") +} + +func TestGitStore_UnsubscribeStopsDelivery(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"a.txt": "1"}, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 16) + id, err := s.Subscribe(t.Context(), + &configuration.SubscribeRequest{}, + makeChanHandler(events), + ) + require.NoError(t, err) + drainOne(t, events) + + require.NoError(t, s.Unsubscribe(t.Context(), &configuration.UnsubscribeRequest{ID: id})) + + u.commit(map[string]string{"a.txt": "2"}, "update") + + // Wait 3× the test pollInterval (1s = 3s total) so the poller has at + // least two opportunities to fire after the commit. Any delivery to a + // cancelled subscriber is a bug; silence here means unsubscribe took + // effect. Inherently timing-dependent — if flaky on slow CI, raise the + // store's pollInterval to slow the cadence in this test. + select { + case e := <-events: + t.Fatalf("unexpected event after unsubscribe: %+v", e) + case <-time.After(3 * time.Second): + } +} + +func TestGitStore_KeyFilter(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{ + "role.txt": "Weather", + "goal.txt": "Help", + }, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 16) + _, err := s.Subscribe(t.Context(), + &configuration.SubscribeRequest{Keys: []string{"role.txt"}}, + makeChanHandler(events), + ) + require.NoError(t, err) + + e := drainOne(t, events) + assert.Len(t, e.Items, 1) + assert.Contains(t, e.Items, "role.txt") + assert.NotContains(t, e.Items, "goal.txt") +} + +func TestGitStore_CloseIdempotent(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"a.txt": "1"}, "seed") + s := newTestStore(t, u, nil) + require.NoError(t, s.Close()) + require.NoError(t, s.Close()) +} + +func TestGitStore_UnsubscribeUnknownID(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"a.txt": "1"}, "seed") + s := newTestStore(t, u, nil) + err := s.Unsubscribe(t.Context(), &configuration.UnsubscribeRequest{ID: "does-not-exist"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not exist") +} + +func TestGitStore_GetAfterCloseFails(t *testing.T) { + u := newUpstream(t) + s := newTestStore(t, u, nil) + require.NoError(t, s.Close()) + _, err := s.Get(t.Context(), &configuration.GetRequest{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "closed", "Get should report the store is closed") +} + +func TestGitStore_SubscribeAfterCloseFails(t *testing.T) { + u := newUpstream(t) + s := newTestStore(t, u, nil) + require.NoError(t, s.Close()) + _, err := s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, func(context.Context, *configuration.UpdateEvent) error { return nil }) + require.Error(t, err) + assert.Contains(t, err.Error(), "closed", "Subscribe should report the store is closed") +} + +func TestGitStore_GetComponentMetadata(t *testing.T) { + s := NewGitConfigurationStore(logger.NewLogger("test")).(*ConfigurationStore) + info := s.GetComponentMetadata() + assert.NotEmpty(t, info) +} + +// TestGitStore_GetReturnsClonedItems asserts that Get returns deep-copied +// items: a caller mutating returned values or metadata must not corrupt the +// store's internal snapshot. Regression test for PR #4380 review feedback. +func TestGitStore_GetReturnsClonedItems(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"k.txt": "original"}, "seed") + s := newTestStore(t, u, nil) + + first, err := s.Get(t.Context(), &configuration.GetRequest{}) + require.NoError(t, err) + require.Contains(t, first.Items, "k.txt") + + // Mutate both the returned item and its metadata map. + first.Items["k.txt"].Value = "MUTATED" + first.Items["k.txt"].Metadata["injected"] = "bad" + + // A subsequent Get must reflect the original snapshot — no carryover. + second, err := s.Get(t.Context(), &configuration.GetRequest{}) + require.NoError(t, err) + assert.Equal(t, "original", second.Items["k.txt"].Value, "store snapshot must be immune to caller mutation") + assert.NotContains(t, second.Items["k.txt"].Metadata, "injected", "store metadata must be immune to caller mutation") +} + +// TestGitStore_SubscribeInitialStateIsCloned asserts that the synchronous +// initial-state event delivered by Subscribe is composed of fresh items — +// mutating an item in the handler must not corrupt the diff base used for +// subsequent ticks. Regression test for PR #4380 review feedback. +func TestGitStore_SubscribeInitialStateIsCloned(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"k.txt": "v1"}, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 4) + mutating := func(_ context.Context, e *configuration.UpdateEvent) error { + // Mutate the delivered item — must not affect the store's snapshot + // or the next diff base. + if v := e.Items["k.txt"]; v != nil { + v.Value = "MUTATED" + if v.Metadata == nil { + v.Metadata = map[string]string{} + } + v.Metadata["injected"] = "bad" + } + events <- e + return nil + } + _, err := s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, mutating) + require.NoError(t, err) + + // Initial state arrives. + drainOne(t, events) + + // Get must reflect the original value, not the mutation done by the handler. + resp, err := s.Get(t.Context(), &configuration.GetRequest{}) + require.NoError(t, err) + assert.Equal(t, "v1", resp.Items["k.txt"].Value, "Subscribe must clone before delivering initial state") + assert.NotContains(t, resp.Items["k.txt"].Metadata, "injected") +} + +func TestGitStore_NoEmitInitialState(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"k.txt": "v"}, "seed") + + s := newTestStore(t, u, map[string]string{"emitInitialState": "false"}) + + events := make(chan *configuration.UpdateEvent, 4) + _, err := s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, makeChanHandler(events)) + require.NoError(t, err) + + // Confirm the poller is alive by triggering a real change. The first event + // the channel sees must be the hot-reload of "v2"; if an initial-state + // event leaked through, it would carry "v" and this assertion would fail. + u.commit(map[string]string{"k.txt": "v2"}, "second commit") + select { + case e := <-events: + v := e.Items["k.txt"] + require.NotNil(t, v, "expected k.txt in update event") + assert.Equal(t, "v2", v.Value, "first delivered event must be the hot-reload, not the initial state") + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for hot-reload confirmation") + } +} + +func TestGitStore_MultiSubscriberFanOut(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"a.txt": "1"}, "seed") + s := newTestStore(t, u, nil) + + ch1 := make(chan *configuration.UpdateEvent, 8) + ch2 := make(chan *configuration.UpdateEvent, 8) + _, err := s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, makeChanHandler(ch1)) + require.NoError(t, err) + _, err = s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, makeChanHandler(ch2)) + require.NoError(t, err) + + drainOne(t, ch1) + drainOne(t, ch2) + + u.commit(map[string]string{"a.txt": "2"}, "update") + + require.Eventually(t, func() bool { + select { + case e := <-ch1: + return e.Items["a.txt"] != nil && e.Items["a.txt"].Value == "2" + default: + return false + } + }, 5*time.Second, 50*time.Millisecond, "subscriber 1 missed the update") + + require.Eventually(t, func() bool { + select { + case e := <-ch2: + return e.Items["a.txt"] != nil && e.Items["a.txt"].Value == "2" + default: + return false + } + }, 5*time.Second, 50*time.Millisecond, "subscriber 2 missed the update") +} + +// TestGitStore_CloseStopsPoller asserts no events arrive after Close(), +// closing the gap left by TestGitStore_CloseIdempotent (which only checks +// Close() returns nil twice). +func TestGitStore_CloseStopsPoller(t *testing.T) { + u := newUpstream(t) + u.commit(map[string]string{"a.txt": "1"}, "seed") + s := newTestStore(t, u, nil) + + events := make(chan *configuration.UpdateEvent, 16) + _, err := s.Subscribe(t.Context(), &configuration.SubscribeRequest{}, makeChanHandler(events)) + require.NoError(t, err) + drainOne(t, events) // initial state + + require.NoError(t, s.Close()) + + // A commit after Close must not be delivered. Wait long enough for at + // least 3 poll intervals to have elapsed if the poller were still alive. + u.commit(map[string]string{"a.txt": "2"}, "post-close commit") + + select { + case e := <-events: + t.Fatalf("unexpected delivery after Close: %+v", e) + case <-time.After(3 * time.Second): + } +} + +// makeChanHandler returns a handler that pushes events to the given channel. +// Errors are NEVER asserted with t.* from inside the handler — that's +// race-prone (Copilot review note in PR #4275). Tests inspect the channel +// from the main goroutine. +func makeChanHandler(ch chan<- *configuration.UpdateEvent) configuration.UpdateHandler { + return func(_ context.Context, e *configuration.UpdateEvent) error { + ch <- e + return nil + } +} + +func drainOne(t *testing.T, ch <-chan *configuration.UpdateEvent) *configuration.UpdateEvent { + t.Helper() + select { + case e := <-ch: + return e + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for event") + return nil + } +} diff --git a/configuration/git/mapping.go b/configuration/git/mapping.go new file mode 100644 index 0000000000..a7509e334c --- /dev/null +++ b/configuration/git/mapping.go @@ -0,0 +1,284 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "bufio" + "bytes" + "fmt" + "path" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/dapr/components-contrib/configuration" + "github.com/dapr/kit/logger" +) + +// extKey returns the lowercased extension of p including the leading ".". +func extKey(p string) string { + return strings.ToLower(path.Ext(p)) +} + +// fileEntry is the input to mapping strategies. RelPath is POSIX-style, +// without a leading slash, relative to the configured Path. +type fileEntry struct { + RelPath string + Bytes []byte +} + +// mappingStrategy converts file entries into a snapshot of configuration +// items. The version string is applied to every Item.Version. +type mappingStrategy interface { + Name() string + Map(entries []fileEntry, version string, log logger.Logger) (map[string]*configuration.Item, error) +} + +func selectMapper(mode string) (mappingStrategy, error) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", mappingModeFile: + return &fileMapper{}, nil + case mappingModeAgentYAML: + return &agentYAMLMapper{}, nil + case mappingModePrompty: + return &promptyMapper{}, nil + } + return nil, fmt.Errorf("unsupported mappingMode %q (supported: file, agentYaml, prompty)", mode) +} + +// fileMapper maps each file to one configuration item with key=relpath, +// value=string(content). +type fileMapper struct{} + +func (fileMapper) Name() string { return mappingModeFile } + +func (fileMapper) Map(entries []fileEntry, version string, _ logger.Logger) (map[string]*configuration.Item, error) { + out := make(map[string]*configuration.Item, len(entries)) + for _, e := range entries { + out[e.RelPath] = &configuration.Item{ + Value: string(e.Bytes), + Version: version, + Metadata: map[string]string{}, + } + } + return out, nil +} + +// agentYAMLMapper parses each *.yaml/*.yml/*.json file as a flat top-level +// map and emits one key per top-level field, prefixed by the filename stem. +// +// Non-matching files in the configured scope are a hard error: if the +// operator selected mappingMode=agentYaml and the path contains a README, +// LICENSE, or any other non-YAML file, that's a configuration mistake the +// operator should be told about — not silently dropped. +type agentYAMLMapper struct{} + +func (agentYAMLMapper) Name() string { return mappingModeAgentYAML } + +func (agentYAMLMapper) Map(entries []fileEntry, version string, log logger.Logger) (map[string]*configuration.Item, error) { + out := make(map[string]*configuration.Item) + owners := make(map[string]string) + for _, e := range entries { + switch extKey(e.RelPath) { + case ".yaml", ".yml", ".json": + default: + return nil, fmt.Errorf("mappingMode=agentYaml does not accept %q: only .yaml/.yml/.json files are permitted; either move the file out of the configured path or switch to mappingMode=file", e.RelPath) + } + stem := stemOf(e.RelPath) + fields, err := parseFlatMap(e.Bytes) + if err != nil { + return nil, fmt.Errorf("mappingMode=agentYaml: parse %q: %w", e.RelPath, err) + } + for k, v := range fields { + key := stem + "/" + k + if owner, exists := owners[key]; exists && log != nil { + log.Warnf("git mapping: key %q from %q overwritten by %q", key, owner, e.RelPath) + } + owners[key] = e.RelPath + out[key] = &configuration.Item{ + Value: v, + Version: version, + Metadata: map[string]string{}, + } + } + } + return out, nil +} + +// promptyMapper splits the YAML frontmatter from the body and emits keys for +// each frontmatter field plus an `agent_system_prompt` carrying the body. +// +// Non-.prompty files in the configured scope are a hard error: mixed-content +// directories must use mappingMode=file. The Prompty spec is at +// https://github.com/microsoft/prompty. +type promptyMapper struct{} + +func (promptyMapper) Name() string { return mappingModePrompty } + +func (promptyMapper) Map(entries []fileEntry, version string, log logger.Logger) (map[string]*configuration.Item, error) { + out := make(map[string]*configuration.Item) + owners := make(map[string]string) + for _, e := range entries { + if extKey(e.RelPath) != ".prompty" { + return nil, fmt.Errorf("mappingMode=prompty does not accept %q: only .prompty files are permitted; either move the file out of the configured path or switch to mappingMode=file", e.RelPath) + } + stem := stemOf(e.RelPath) + frontmatter, body, err := splitPromptyFrontmatter(e.Bytes) + if err != nil { + return nil, fmt.Errorf("mappingMode=prompty: %q: %w", e.RelPath, err) + } + if len(frontmatter) > 0 { + fields, err := parseFlatMap(frontmatter) + if err != nil { + return nil, fmt.Errorf("mappingMode=prompty: parse frontmatter of %q: %w", e.RelPath, err) + } + for k, v := range fields { + key := stem + "/" + k + if owner, exists := owners[key]; exists && log != nil { + log.Warnf("git mapping: key %q from %q overwritten by %q", key, owner, e.RelPath) + } + owners[key] = e.RelPath + out[key] = &configuration.Item{ + Value: v, + Version: version, + Metadata: map[string]string{}, + } + } + } + body = bytes.TrimSpace(body) + if len(body) > 0 { + key := stem + "/agent_system_prompt" + if owner, exists := owners[key]; exists && log != nil { + log.Warnf("git mapping: key %q from %q overwritten by %q", key, owner, e.RelPath) + } + owners[key] = e.RelPath + out[key] = &configuration.Item{ + Value: string(body), + Version: version, + Metadata: map[string]string{}, + } + } + } + return out, nil +} + +// stemOf returns a deterministic, collision-free key prefix derived from the +// relative path. Directory separators are replaced with `_` so two files with +// the same basename in different subdirectories produce different stems. +// The result is lowercased to keep keys stable across case-insensitive +// filesystems. Examples: +// +// "weather.yaml" → "weather" +// "agents/weather.yaml" → "agents_weather" +// "team-a/weather.prompty" → "team-a_weather" +func stemOf(p string) string { + if i := strings.LastIndex(p, "."); i > 0 { + p = p[:i] + } + return strings.ToLower(strings.ReplaceAll(p, "/", "_")) +} + +// parseFlatMap parses YAML/JSON bytes as a top-level map[string]any and +// returns string-encoded values: scalars via fmt, non-scalars via canonical +// YAML serialisation (lossless round-trip for the consumer). +func parseFlatMap(b []byte) (map[string]string, error) { + var raw map[string]any + if err := yaml.Unmarshal(b, &raw); err != nil { + return nil, fmt.Errorf("yaml decode: %w", err) + } + out := make(map[string]string, len(raw)) + for k, v := range raw { + s, err := encodeFieldValue(v) + if err != nil { + return nil, fmt.Errorf("encode field %q: %w", k, err) + } + out[k] = s + } + return out, nil +} + +func encodeFieldValue(v any) (string, error) { + switch x := v.(type) { + case nil: + return "", nil + case string: + return x, nil + case bool: + if x { + return "true", nil + } + return "false", nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return fmt.Sprintf("%v", x), nil + } + // Maps, slices, or anything else — re-serialise as YAML so the consumer + // can round-trip without losing structure. + out, err := yaml.Marshal(v) + if err != nil { + return "", err + } + return strings.TrimRight(string(out), "\n"), nil +} + +// splitPromptyFrontmatter accepts a prompty file and returns (frontmatter, +// body). The grammar is: +// +// ---\n +// \n +// ---\n +// +// +// Files without a leading `---` are treated as body-only. Files with only the +// opening `---` (and no closing) are treated as frontmatter-only — the +// remainder is yaml. Returns an error if the file is too large for the +// scanner buffer (4 MiB) so the caller can skip rather than silently produce +// a truncated result. +func splitPromptyFrontmatter(b []byte) (frontmatter, body []byte, err error) { + scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner.Buffer(make([]byte, 64*1024), 4*1024*1024) + lines := make([][]byte, 0) + for scanner.Scan() { + // Copy the slice — bufio's buffer is re-used. + raw := scanner.Bytes() + clone := make([]byte, len(raw)) + copy(clone, raw) + lines = append(lines, clone) + } + if scanErr := scanner.Err(); scanErr != nil { + return nil, nil, fmt.Errorf("scan prompty file: %w", scanErr) + } + if len(lines) == 0 { + return nil, nil, nil + } + if !isFrontmatterDelim(lines[0]) { + return nil, b, nil + } + for i := 1; i < len(lines); i++ { + if isFrontmatterDelim(lines[i]) { + frontmatter = bytes.Join(lines[1:i], []byte("\n")) + if i+1 < len(lines) { + body = bytes.Join(lines[i+1:], []byte("\n")) + } + return frontmatter, body, nil + } + } + // Closing delimiter not found — treat all content after the opening + // delimiter as frontmatter. + frontmatter = bytes.Join(lines[1:], []byte("\n")) + return frontmatter, nil, nil +} + +func isFrontmatterDelim(line []byte) bool { + return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---")) +} diff --git a/configuration/git/mapping_test.go b/configuration/git/mapping_test.go new file mode 100644 index 0000000000..391bee501a --- /dev/null +++ b/configuration/git/mapping_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dapr/kit/logger" +) + +const testVersion = "abc1234" + +func TestSelectMapper(t *testing.T) { + tests := []struct { + name string + mode string + want string + wantErr bool + }{ + {"empty defaults to file", "", mappingModeFile, false}, + {"lowercase file", "file", mappingModeFile, false}, + {"uppercase FILE", "FILE", mappingModeFile, false}, + {"agentYaml mixed case", "agentYaml", mappingModeAgentYAML, false}, + {"prompty", "prompty", mappingModePrompty, false}, + {"unsupported mode rejected", "bogus", "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := selectMapper(tt.mode) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, m.Name()) + }) + } +} + +func TestFileMapper(t *testing.T) { + mapper := fileMapper{} + entries := []fileEntry{ + {RelPath: "agent_role.txt", Bytes: []byte("Weather expert")}, + {RelPath: "nested/agent_goal.txt", Bytes: []byte("Help users")}, + {RelPath: "instructions.json", Bytes: []byte(`["be concise"]`)}, + } + out, err := mapper.Map(entries, testVersion, logger.NewLogger("test")) + require.NoError(t, err) + require.Len(t, out, 3) + + assert.Equal(t, "Weather expert", out["agent_role.txt"].Value) + assert.Equal(t, testVersion, out["agent_role.txt"].Version) + assert.Equal(t, "Help users", out["nested/agent_goal.txt"].Value) + assert.Equal(t, `["be concise"]`, out["instructions.json"].Value) +} + +func TestAgentYAMLMapper(t *testing.T) { + mapper := agentYAMLMapper{} + yamlInput := []byte(`agent_role: Weather expert +agent_goal: Help users plan trips +agent_instructions: + - be concise + - cite sources +max_iterations: 5 +`) + jsonInput := []byte(`{"agent_role": "Travel agent", "tool_choice": "auto"}`) + + entries := []fileEntry{ + {RelPath: "weather.yaml", Bytes: yamlInput}, + {RelPath: "travel.json", Bytes: jsonInput}, + } + out, err := mapper.Map(entries, testVersion, logger.NewLogger("test")) + require.NoError(t, err) + + assert.Equal(t, "Weather expert", out["weather/agent_role"].Value) + assert.Equal(t, "Help users plan trips", out["weather/agent_goal"].Value) + assert.Equal(t, "5", out["weather/max_iterations"].Value) + + // Non-scalar lossless round-trip. + require.Contains(t, out, "weather/agent_instructions") + roundtrip := out["weather/agent_instructions"].Value + assert.Contains(t, roundtrip, "be concise") + assert.Contains(t, roundtrip, "cite sources") + + assert.Equal(t, "Travel agent", out["travel/agent_role"].Value) + assert.Equal(t, "auto", out["travel/tool_choice"].Value) +} + +func TestAgentYAMLMapper_NonYAMLFileRejected(t *testing.T) { + // mappingMode=agentYaml must hard-error on non-yaml/json files so the + // operator knows their `path` scope and mappingMode are mismatched. + mapper := agentYAMLMapper{} + entries := []fileEntry{ + {RelPath: "weather.yaml", Bytes: []byte("agent_role: ok\n")}, + {RelPath: "README.md", Bytes: []byte("not yaml")}, + } + _, err := mapper.Map(entries, testVersion, logger.NewLogger("test")) + require.Error(t, err) + assert.Contains(t, err.Error(), "mappingMode=agentYaml does not accept \"README.md\"") +} + +func TestAgentYAMLMapper_SubdirectoryStemUniqueness(t *testing.T) { + // Two files with the same basename in different subdirectories must not + // collide on emitted keys. This guards against the historical stemOf + // behaviour that discarded directory components. + mapper := agentYAMLMapper{} + entries := []fileEntry{ + {RelPath: "team-a/weather.yaml", Bytes: []byte("agent_role: Weather A\n")}, + {RelPath: "team-b/weather.yaml", Bytes: []byte("agent_role: Weather B\n")}, + } + out, err := mapper.Map(entries, testVersion, logger.NewLogger("test")) + require.NoError(t, err) + assert.Equal(t, "Weather A", out["team-a_weather/agent_role"].Value) + assert.Equal(t, "Weather B", out["team-b_weather/agent_role"].Value) +} + +func TestAgentYAMLMapper_BadFileRejected(t *testing.T) { + // A YAML file that doesn't decode into a top-level map is a structural + // mismatch: the operator told the component to expect agent-shaped YAML. + // Hard-error so they notice rather than silently dropping their config. + mapper := agentYAMLMapper{} + entries := []fileEntry{ + {RelPath: "bad.yaml", Bytes: []byte("- not a map\n- but a sequence\n")}, + } + _, err := mapper.Map(entries, testVersion, logger.NewLogger("test")) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse \"bad.yaml\"") +} + +func TestPromptyMapper(t *testing.T) { + full := []byte(`--- +name: Weather +agent_role: Weather expert +agent_goal: Help users plan trips +agent_instructions: + - be concise +--- +You are a friendly weather assistant. +`) + bodyOnly := []byte(`Just a system prompt with no frontmatter. +`) + frontmatterOnly := []byte(`--- +agent_role: Travel agent +--- +`) + + entries := []fileEntry{ + {RelPath: "weather.prompty", Bytes: full}, + {RelPath: "raw.prompty", Bytes: bodyOnly}, + {RelPath: "travel.prompty", Bytes: frontmatterOnly}, + } + out, err := promptyMapper{}.Map(entries, testVersion, logger.NewLogger("test")) + require.NoError(t, err) + + assert.Equal(t, "Weather expert", out["weather/agent_role"].Value) + assert.Equal(t, "Help users plan trips", out["weather/agent_goal"].Value) + assert.Equal(t, "You are a friendly weather assistant.", + strings.TrimSpace(out["weather/agent_system_prompt"].Value)) + + require.Contains(t, out, "raw/agent_system_prompt") + assert.Contains(t, out["raw/agent_system_prompt"].Value, "Just a system prompt") + _, hasRawFrontmatter := out["raw/agent_role"] + assert.False(t, hasRawFrontmatter) + + assert.Equal(t, "Travel agent", out["travel/agent_role"].Value) + _, hasTravelBody := out["travel/agent_system_prompt"] + assert.False(t, hasTravelBody) +} + +func TestPromptyMapper_NonPromptyFileRejected(t *testing.T) { + // mappingMode=prompty must hard-error on non-.prompty files. + entries := []fileEntry{ + {RelPath: "weather.prompty", Bytes: []byte("---\nagent_role: ok\n---\n")}, + {RelPath: "ignored.txt", Bytes: []byte("nope")}, + } + _, err := promptyMapper{}.Map(entries, testVersion, logger.NewLogger("test")) + require.Error(t, err) + assert.Contains(t, err.Error(), "mappingMode=prompty does not accept \"ignored.txt\"") +} + +func TestPromptyMapper_NoClosingDelim(t *testing.T) { + // No closing `---` — everything after the opener is frontmatter, no body. + in := []byte(`--- +agent_role: Recovered +agent_goal: From malformed input +`) + out, err := promptyMapper{}.Map( + []fileEntry{{RelPath: "fallback.prompty", Bytes: in}}, + testVersion, logger.NewLogger("test"), + ) + require.NoError(t, err) + assert.Equal(t, "Recovered", out["fallback/agent_role"].Value) + _, hasBody := out["fallback/agent_system_prompt"] + assert.False(t, hasBody) +} diff --git a/configuration/git/metadata.go b/configuration/git/metadata.go new file mode 100644 index 0000000000..4f46c405ba --- /dev/null +++ b/configuration/git/metadata.go @@ -0,0 +1,358 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "errors" + "fmt" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/dapr/components-contrib/configuration" + kitmd "github.com/dapr/kit/metadata" +) + +const ( + mappingModeFile = "file" + mappingModeAgentYAML = "agentyaml" + mappingModePrompty = "prompty" + + // authMode* constants are internal sentinels used by the auto-detector + // and downstream auth strategy selection. They are not user-facing — + // operators do not select a mode explicitly; it is derived from which + // metadata fields are populated (see resolveAuthMode). + authModeNone = "none" + authModePAT = "pat" + authModeSSH = "ssh" + authModeGithubApp = "githubapp" + + defaultBranch = "main" + defaultPath = "." + defaultDepth = 0 + defaultPollInterval = 5 * time.Minute + defaultFetchTimeout = 30 * time.Second + defaultMaxFileSizeBytes = int64(1 * 1024 * 1024) // 1 MiB per file + defaultSnapshotCacheSize = 4 + defaultMappingMode = mappingModeFile + defaultUser = "git" + defaultAPIBase = "https://api.github.com" + defaultRefreshSkew = 5 * time.Minute + defaultRateLimitRetryAfter = 5 * time.Minute + // minPollInterval is the absolute floor for remote (https/ssh) URLs to + // guard the operator's git provider rate-limit budget. file:// URLs + // bypass this floor since they have no rate-limit concern. + minPollInterval = time.Second + minLocalPollInterval = 100 * time.Millisecond +) + +// metadata is the decoded form of the Dapr component spec for +// configuration.git. Field names use the mapstructure tag to match the +// user-facing YAML keys. There is no `authMode` selector — the active auth +// profile is inferred from which fields are set (see resolveAuthMode). +type metadata struct { + RemoteURL string `mapstructure:"remoteUrl"` // required + + Branch *string `mapstructure:"branch"` + Path *string `mapstructure:"path"` + Depth *int `mapstructure:"depth"` + PollInterval *time.Duration `mapstructure:"pollInterval"` + FetchTimeout *time.Duration `mapstructure:"fetchTimeout"` + IncludeHidden *bool `mapstructure:"includeHidden"` + EmitInitialState *bool `mapstructure:"emitInitialState"` + MaxFileSize *int64 `mapstructure:"maxFileSize"` + SnapshotCacheSize *int `mapstructure:"snapshotCacheSize"` + RateLimitRetryAfter *time.Duration `mapstructure:"rateLimitRetryAfter"` + + MappingMode *string `mapstructure:"mappingMode"` + + // Personal Access Token profile + Username *string `mapstructure:"username"` + Token string `mapstructure:"token"` + + // SSH profile. PrivateKey/PrivateKeyPath are also reused by the GitHub + // App profile — auto-detection picks one based on AppID presence. + User *string `mapstructure:"user"` + Passphrase string `mapstructure:"passphrase"` + KnownHosts *string `mapstructure:"knownHosts"` + KnownHostsPath *string `mapstructure:"knownHostsPath"` + InsecureIgnoreHostKey *bool `mapstructure:"insecureIgnoreHostKey"` + + // GitHub App profile + AppID *int64 `mapstructure:"appId"` + InstallationID *int64 `mapstructure:"installationId"` + APIBase *string `mapstructure:"apiBase"` + RefreshSkew *time.Duration `mapstructure:"refreshSkew"` + + // Shared between SSH and GitHub App profiles. + PrivateKey string `mapstructure:"privateKey"` + PrivateKeyPath *string `mapstructure:"privateKeyPath"` +} + +func (m *metadata) parse(meta configuration.Metadata) error { + if err := kitmd.DecodeMetadata(meta.Properties, m); err != nil { + return fmt.Errorf("failed to decode metadata: %w", err) + } + + if strings.TrimSpace(m.RemoteURL) == "" { + return errors.New("remoteUrl is required") + } + + mode := normalizeMode(m.MappingMode) + switch mode { + case "", mappingModeFile, mappingModeAgentYAML, mappingModePrompty: + default: + return fmt.Errorf("unsupported mappingMode %q (supported: file, agentYaml, prompty)", *m.MappingMode) + } + + resolved := m.resolveAuthMode() + if err := m.validateAuthFields(resolved); err != nil { + return err + } + if err := m.validateURL(resolved); err != nil { + return err + } + + if m.PollInterval != nil { + floor := minPollInterval + if strings.HasPrefix(m.RemoteURL, "file://") { + floor = minLocalPollInterval + } + if *m.PollInterval < floor { + return fmt.Errorf("pollInterval %s is below minimum of %s", *m.PollInterval, floor) + } + } + + if m.Path != nil { + clean := filepath.ToSlash(filepath.Clean(*m.Path)) + if strings.HasPrefix(clean, "/") { + return fmt.Errorf("path %q must be repo-relative (no leading slash)", *m.Path) + } + if clean == ".." || strings.HasPrefix(clean, "../") { + return fmt.Errorf("path %q must not escape the repository root", *m.Path) + } + // Reject any path segment equal to ".git" so the walker never sees + // a scope that descends into the git metadata directory. The walker + // also refuses to enter `.git` as a backstop (see walk.go), but + // catching it at validation gives a clear error message instead of + // a silently empty snapshot. + for _, seg := range strings.Split(clean, "/") { + if seg == ".git" { + return fmt.Errorf("path %q must not reference the .git directory", *m.Path) + } + } + } + + return nil +} + +func (m *metadata) branch() string { + if m.Branch != nil && *m.Branch != "" { + return *m.Branch + } + return defaultBranch +} + +func (m *metadata) path() string { + if m.Path != nil && *m.Path != "" { + return *m.Path + } + return defaultPath +} + +func (m *metadata) depth() int { + if m.Depth != nil { + return *m.Depth + } + return defaultDepth +} + +func (m *metadata) pollInterval() time.Duration { + if m.PollInterval != nil { + return *m.PollInterval + } + return defaultPollInterval +} + +func (m *metadata) fetchTimeout() time.Duration { + if m.FetchTimeout != nil { + return *m.FetchTimeout + } + return defaultFetchTimeout +} + +func (m *metadata) includeHidden() bool { + if m.IncludeHidden != nil { + return *m.IncludeHidden + } + return false +} + +func (m *metadata) emitInitialState() bool { + if m.EmitInitialState != nil { + return *m.EmitInitialState + } + return true +} + +func (m *metadata) maxFileSize() int64 { + if m.MaxFileSize != nil && *m.MaxFileSize > 0 { + return *m.MaxFileSize + } + return defaultMaxFileSizeBytes +} + +func (m *metadata) snapshotCacheSize() int { + if m.SnapshotCacheSize != nil && *m.SnapshotCacheSize > 0 { + return *m.SnapshotCacheSize + } + return defaultSnapshotCacheSize +} + +func (m *metadata) rateLimitRetryAfter() time.Duration { + if m.RateLimitRetryAfter != nil && *m.RateLimitRetryAfter > 0 { + return *m.RateLimitRetryAfter + } + return defaultRateLimitRetryAfter +} + +func (m *metadata) mappingMode() string { + if mode := normalizeMode(m.MappingMode); mode != "" { + return mode + } + return defaultMappingMode +} + +func (m *metadata) user() string { + if m.User != nil && *m.User != "" { + return *m.User + } + return defaultUser +} + +func (m *metadata) insecureIgnoreHostKey() bool { + if m.InsecureIgnoreHostKey != nil { + return *m.InsecureIgnoreHostKey + } + return false +} + +func (m *metadata) apiBase() string { + if m.APIBase != nil && *m.APIBase != "" { + return *m.APIBase + } + return defaultAPIBase +} + +func (m *metadata) refreshSkew() time.Duration { + if m.RefreshSkew != nil { + return *m.RefreshSkew + } + return defaultRefreshSkew +} + +// resolveAuthMode derives which auth profile is active from the metadata +// the operator provided. There is no explicit selector — presence of +// fields determines the profile. Priority: +// +// 1. appId set → GitHub App +// 2. URL is SSH-scheme (git@ or ssh://) → SSH +// 3. token set → PAT +// 4. otherwise → no auth (public HTTPS or file://) +func (m *metadata) resolveAuthMode() string { + if m.AppID != nil && *m.AppID != 0 { + return authModeGithubApp + } + if strings.HasPrefix(m.RemoteURL, "git@") || strings.HasPrefix(m.RemoteURL, "ssh://") { + return authModeSSH + } + if m.Token != "" { + return authModePAT + } + return authModeNone +} + +func (m *metadata) validateAuthFields(mode string) error { + switch mode { + case authModeNone: + return nil + case authModePAT: + if m.Token == "" { + return errors.New("PAT auth requires token") + } + return nil + case authModeSSH: + if m.PrivateKey == "" && (m.PrivateKeyPath == nil || *m.PrivateKeyPath == "") { + return errors.New("SSH auth requires privateKey or privateKeyPath") + } + return nil + case authModeGithubApp: + if m.AppID == nil || *m.AppID == 0 { + return errors.New("GitHub App auth requires appId") + } + if m.InstallationID == nil || *m.InstallationID == 0 { + return errors.New("GitHub App auth requires installationId") + } + if m.PrivateKey == "" && (m.PrivateKeyPath == nil || *m.PrivateKeyPath == "") { + return errors.New("GitHub App auth requires privateKey or privateKeyPath") + } + if m.APIBase != nil && *m.APIBase != "" { + u, err := url.Parse(*m.APIBase) + if err != nil { + return fmt.Errorf("apiBase: %w", err) + } + if u.Scheme != "https" { + return errors.New("apiBase must use https://") + } + } + return nil + } + return fmt.Errorf("unsupported auth mode %q", mode) +} + +// validateURL rejects insecure schemes (http://) for authenticated profiles +// and rejects URLs that embed credentials inline so operators are forced to +// store them in a secret store. SCP-style SSH URLs (git@host:path) and local +// file:// URLs are recognised explicitly: the former doesn't parse cleanly +// via url.Parse, and the latter may legitimately contain Windows drive +// letters (e.g. file:///C:/Users/...) that net/url interprets as host:port. +func (m *metadata) validateURL(resolvedAuth string) error { + if strings.HasPrefix(m.RemoteURL, "git@") { + return nil + } + if strings.HasPrefix(m.RemoteURL, "file://") { + return nil + } + parsed, err := url.Parse(m.RemoteURL) + if err != nil { + return fmt.Errorf("remoteUrl: %w", err) + } + if parsed.User != nil && parsed.User.Username() != "" { + if _, hasPass := parsed.User.Password(); hasPass { + return errors.New("remoteUrl: credentials must not be embedded in the URL; supply them via the appropriate auth profile field (typically backed by a configured secret store)") + } + } + if resolvedAuth != authModeNone && parsed.Scheme == "http" { + return errors.New("remoteUrl: http:// is not allowed with an authenticated profile; use https://, ssh://, or file://") + } + return nil +} + +func normalizeMode(s *string) string { + if s == nil { + return "" + } + return strings.ToLower(strings.TrimSpace(*s)) +} diff --git a/configuration/git/metadata.yaml b/configuration/git/metadata.yaml new file mode 100644 index 0000000000..26ea5adc12 --- /dev/null +++ b/configuration/git/metadata.yaml @@ -0,0 +1,293 @@ +# yaml-language-server: $schema=../../component-metadata-schema.json +schemaVersion: v1 +type: configuration +name: git +version: v1 +status: alpha +title: "Git" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-configuration-stores/git-configuration-store/ +capabilities: [] +authenticationProfiles: + - title: "Personal Access Token" + description: | + HTTPS basic auth using a token. Works with both GitHub classic PATs + (`ghp_…`) and fine-grained PATs (`github_pat_…`). + metadata: + - name: token + required: true + sensitive: true + description: | + The personal access token used to authenticate. The token is sent + as the password in HTTP basic auth. Source from a configured + secret store. + example: "ghp_xxxxxxxxxxxxxxxxxxxx" + type: string + - name: username + required: false + description: | + The username sent with the token. Defaults to "x-access-token", + which is recommended by GitHub when using a PAT. Other providers + may require a real username. + example: "x-access-token" + default: "x-access-token" + type: string + - title: "SSH" + description: "Authenticate over SSH with a private key." + metadata: + - name: user + required: false + description: | + SSH user used when connecting. Defaults to "git", which matches + the convention used by GitHub, GitLab, Bitbucket, and most + self-hosted git providers. Override for self-hosted providers + that gate SSH login on a non-"git" username. + example: "deploy-user" + default: "git" + type: string + - name: privateKey + required: false + sensitive: true + description: | + PEM-encoded SSH private key. Either privateKey or privateKeyPath + must be set. + example: "-----BEGIN OPENSSH PRIVATE KEY-----\n..." + type: string + - name: privateKeyPath + required: false + description: | + Path to a PEM-encoded SSH private key on disk. Either privateKey + or privateKeyPath must be set. + example: "/var/run/secrets/git-ssh-key" + type: string + - name: passphrase + required: false + sensitive: true + description: "Passphrase for the SSH private key, if encrypted." + example: "" + type: string + - name: knownHosts + required: false + description: | + Inline OpenSSH known_hosts entries used to verify the remote host + key. Either knownHosts or knownHostsPath must be set unless + insecureIgnoreHostKey is true. + example: "github.com ssh-rsa AAAA..." + type: string + - name: knownHostsPath + required: false + description: "Path to an OpenSSH known_hosts file on disk." + example: "/etc/ssh/ssh_known_hosts" + type: string + - name: insecureIgnoreHostKey + required: false + description: | + Disable SSH host key verification. DANGEROUS — only use for local + development or testing. When enabled, the component logs a + warning at startup. Never enable in production; a MITM attacker + can intercept configuration values. + example: "false" + default: "false" + type: bool + - title: "GitHub App" + description: | + Authenticate via a GitHub App installation. The component mints a + JWT, exchanges it for a 1-hour installation token, and refreshes the + token before expiry. On HTTP 429 (or 403 with rate-limit headers) + the component honours Retry-After and retries once; persistent + rate-limit responses are surfaced to the poll loop which backs off + for `rateLimitRetryAfter` before its next tick. + metadata: + - name: appId + required: true + description: "Numeric GitHub App ID." + example: "123456" + type: number + - name: installationId + required: true + description: | + Numeric GitHub App installation ID for the target organisation + or repository. + example: "78901234" + type: number + - name: privateKey + required: false + sensitive: true + description: | + PEM-encoded RSA private key for the GitHub App. Accepts both + PKCS#1 ("RSA PRIVATE KEY") and PKCS#8 ("PRIVATE KEY") encodings. + Either privateKey or privateKeyPath must be set. + example: "-----BEGIN RSA PRIVATE KEY-----\n..." + type: string + - name: privateKeyPath + required: false + description: "Path to a PEM-encoded RSA private key on disk." + example: "/var/run/secrets/github-app-key.pem" + type: string + - name: apiBase + required: false + description: | + Base URL of the GitHub API. Override for GitHub Enterprise + Server. Must use https://. + example: "https://api.github.com" + default: "https://api.github.com" + type: string + - name: refreshSkew + required: false + description: | + Refresh the installation token when it has less than this much + time left before expiry. + example: "5m" + default: "5m" + type: duration +metadata: + - name: remoteUrl + required: true + description: | + Git URL of the upstream remote (the same value `git remote get-url + origin` would return for a clone). Supports https://, ssh://, git@, + and file:// schemes. URLs with the git@ or ssh:// prefix auto-select + the SSH auth profile. http:// is rejected when any authenticated + profile is in use to prevent cleartext credential transmission. + Embedding credentials in the URL (e.g. https://user:token@host) is + rejected; supply them via the appropriate auth profile field backed + by a configured secret store. + example: "https://github.com/example/agent-config.git" + type: string + - name: branch + required: false + description: "Branch to track." + example: "main" + default: "main" + type: string + - name: path + required: false + description: | + Subdirectory inside the repository to treat as the configuration + root. Files outside this directory are not surfaced as configuration + items. Must be repo-relative (no leading "/", no ".." components, + and no segment may be ".git"). Default "." means "use the + repository root". + example: "agents/weather" + default: "." + type: string + - name: depth + required: false + description: | + Clone depth. 0 (default) performs a full clone, matching git's + default behaviour when no `--depth` is passed. go-git's shallow + incremental fetch has known limitations; full clones are the safe + choice for anything but trivial config repos. + example: "0" + default: "0" + type: number + - name: pollInterval + required: false + description: | + How often to poll the upstream for changes. Hard floor is 1s for + remote URLs; file:// URLs may go down to 100ms. Intervals below 5s + log a warning at startup. Default 5m gives plenty of head-room + against provider rate limits even on multi-replica deployments — + at 5m × 10 replicas you issue 120 requests/h, well below GitHub's + 5000/h PAT and 15000/h GitHub App limits. If the upstream responds + with HTTP 429 or a transport-level rate-limit error, the poll loop + pauses for rateLimitRetryAfter (or the server-supplied Retry-After + when available) before its next tick. + example: "5m" + default: "5m" + type: duration + - name: rateLimitRetryAfter + required: false + description: | + How long the poll loop waits before its next tick when the upstream + responds with a rate-limit error and no Retry-After header was + supplied. Tune this if you are hitting secondary rate limits on a + busy multi-replica deployment. + example: "5m" + default: "5m" + type: duration + - name: fetchTimeout + required: false + description: "Per-fetch timeout applied to fetch operations." + example: "30s" + default: "30s" + type: duration + - name: includeHidden + required: false + description: | + When false (default), files whose name begins with "." are skipped + during the worktree walk. The ".git" directory is always excluded + regardless of this flag — it would otherwise expose remote URLs and + possibly embedded credentials as configuration items. + example: "false" + default: "false" + type: bool + - name: maxFileSize + required: false + description: | + Maximum per-file size in bytes that the walker will read into + memory. Files larger than this are skipped with a warning. Protects + the sidecar from OOM if a large blob is accidentally committed to + the configuration repo. Default is 1 MiB (1048576 bytes). + example: "1048576" + default: "1048576" + type: number + - name: snapshotCacheSize + required: false + description: | + Number of past snapshots to retain in the LRU cache used as diff + bases when computing per-subscriber update events. Higher values + reduce over-emit churn when many subscribers are at slightly + different delivered HEAD values. With the default pollInterval of + 5m, the default of 4 covers ~20 minutes of commit history for the + most-stale subscriber — sufficient for typical agent deployments. + Raise it if you run a large pool of subscribers against a busy + configuration repo. + example: "4" + default: "4" + type: number + - name: emitInitialState + required: false + description: | + When true (default), Subscribe synchronously delivers the current + snapshot to the handler before returning so callers don't need a + separate Get + Subscribe pair. Set to false in tests or when the + caller already has fresh state. + example: "true" + default: "true" + type: bool + - name: mappingMode + required: false + description: | + Strategy for mapping files in the repository to configuration + items. Matching is case-insensitive. Non-matching files are a + hard error — if your scope contains a mix of file types, either + narrow `path` to the homogeneous subset or use mappingMode=file. + + The `agentYaml` and `prompty` modes target dapr-agents-style + configuration repos. `prompty` follows the Prompty spec + (https://github.com/microsoft/prompty) for `.prompty` files. + + - file (default): each file is one config item. Key = relative + POSIX path, value = file contents as string. + - agentYaml: a convenience mode for repos that store agent + definitions as YAML/JSON. Each *.yaml/*.yml/*.json file is + parsed as a flat top-level map; each top-level field becomes a + key prefixed by the filename stem with directory separators + replaced by `_`, e.g. "agents/weather.yaml" field "agent_role" + → "agents_weather/agent_role". The name describes the intended + layout, not an external spec. Non-YAML/JSON files in scope + cause Init to fail. + - prompty: each *.prompty file's YAML frontmatter and body are + split per the Prompty spec linked above. Frontmatter fields + produce "/" keys (same stem rules as agentYaml); + the body is emitted as "/agent_system_prompt". + Non-.prompty files in scope cause Init to fail. + example: "file" + default: "file" + type: string + allowedValues: + - "file" + - "agentYaml" + - "prompty" diff --git a/configuration/git/metadata_test.go b/configuration/git/metadata_test.go new file mode 100644 index 0000000000..3d8f8bd65d --- /dev/null +++ b/configuration/git/metadata_test.go @@ -0,0 +1,290 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dapr/components-contrib/configuration" + contribMetadata "github.com/dapr/components-contrib/metadata" +) + +func TestMetadata_Parse(t *testing.T) { + tests := []struct { + name string + props map[string]string + wantErr string + check func(t *testing.T, m *metadata) + }{ + { + name: "missing remoteUrl", + props: map[string]string{}, + wantErr: "remoteUrl is required", + }, + { + name: "defaults applied", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, "main", m.branch()) + assert.Equal(t, ".", m.path()) + assert.Equal(t, 0, m.depth()) + assert.Equal(t, 5*time.Minute, m.pollInterval()) + assert.Equal(t, 30*time.Second, m.fetchTimeout()) + assert.False(t, m.includeHidden()) + assert.Equal(t, mappingModeFile, m.mappingMode()) + assert.Equal(t, authModeNone, m.resolveAuthMode()) + assert.Equal(t, "git", m.user()) + assert.Equal(t, "https://api.github.com", m.apiBase()) + assert.Equal(t, 5*time.Minute, m.refreshSkew()) + assert.Equal(t, 5*time.Minute, m.rateLimitRetryAfter()) + }, + }, + { + name: "explicit overrides", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "branch": "release", + "path": "agents", + "depth": "5", + "pollInterval": "15s", + "fetchTimeout": "10s", + "includeHidden": "true", + "mappingMode": "agentYaml", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, "release", m.branch()) + assert.Equal(t, "agents", m.path()) + assert.Equal(t, 5, m.depth()) + assert.Equal(t, 15*time.Second, m.pollInterval()) + assert.Equal(t, 10*time.Second, m.fetchTimeout()) + assert.True(t, m.includeHidden()) + //nolint:testifylint // not a YAML payload comparison; mode constant happens to contain "yaml" + assert.Equal(t, mappingModeAgentYAML, m.mappingMode()) + }, + }, + { + name: "unsupported mappingMode", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "mappingMode": "bogus", + }, + wantErr: `unsupported mappingMode "bogus"`, + }, + { + name: "auto-detect ssh from URL", + props: map[string]string{ + "remoteUrl": "git@github.com:org/repo.git", + "privateKeyPath": "/etc/ssh/key", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, authModeSSH, m.resolveAuthMode()) + }, + }, + { + name: "auto-detect ssh missing key", + props: map[string]string{ + "remoteUrl": "git@github.com:org/repo.git", + }, + wantErr: "SSH auth requires privateKey or privateKeyPath", + }, + { + name: "auto-detect pat from token", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "token": "ghp_xxx", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, authModePAT, m.resolveAuthMode()) + }, + }, + { + name: "githubApp missing installationId", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "appId": "1", + "privateKey": "PEM", + }, + wantErr: "installationId", + }, + { + name: "githubApp full config", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "appId": "100", + "installationId": "200", + "privateKey": "PEM", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, authModeGithubApp, m.resolveAuthMode()) + require.NotNil(t, m.AppID) + assert.Equal(t, int64(100), *m.AppID) + require.NotNil(t, m.InstallationID) + assert.Equal(t, int64(200), *m.InstallationID) + }, + }, + { + name: "pollInterval below remote minimum", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "pollInterval": "500ms", + }, + wantErr: "below minimum", + }, + { + name: "pollInterval allowed for file:// below remote minimum", + props: map[string]string{ + "remoteUrl": "file:///tmp/repo.git", + "pollInterval": "500ms", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, 500*time.Millisecond, m.pollInterval()) + }, + }, + { + name: "pollInterval below file minimum", + props: map[string]string{ + "remoteUrl": "file:///tmp/repo.git", + "pollInterval": "10ms", + }, + wantErr: "below minimum", + }, + { + name: "url with embedded credentials rejected", + props: map[string]string{ + "remoteUrl": "https://user:tok@example.com/repo.git", + }, + wantErr: "credentials must not be embedded", + }, + { + name: "http url rejected with authenticated profile", + props: map[string]string{ + "remoteUrl": "http://example.com/repo.git", + "token": "abc", + }, + wantErr: "http:// is not allowed", + }, + { + name: "apiBase requires https", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "appId": "1", + "installationId": "2", + "privateKey": "PEM", + "apiBase": "http://api.example/", + }, + wantErr: "apiBase must use https", + }, + { + name: "path with leading slash", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": "/etc", + }, + wantErr: "must be repo-relative", + }, + { + name: "path escaping repo root", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": "../outside", + }, + wantErr: "must not escape", + }, + { + name: "path equal to .git rejected", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": ".git", + }, + wantErr: "must not reference the .git directory", + }, + { + name: "path with .git subdirectory segment rejected", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": "agents/.git", + }, + wantErr: "must not reference the .git directory", + }, + { + name: "path reaching into .git rejected", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": ".git/config", + }, + wantErr: "must not reference the .git directory", + }, + { + name: "path with .git in middle rejected", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "path": "a/.git/b", + }, + wantErr: "must not reference the .git directory", + }, + { + name: "rateLimitRetryAfter custom", + props: map[string]string{ + "remoteUrl": "https://example.com/repo.git", + "rateLimitRetryAfter": "15m", + }, + check: func(t *testing.T, m *metadata) { + assert.Equal(t, 15*time.Minute, m.rateLimitRetryAfter()) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &metadata{} + err := m.parse(configuration.Metadata{Base: contribMetadata.Base{Properties: tt.props}}) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + if tt.check != nil { + tt.check(t, m) + } + }) + } +} + +func TestMetadata_NormalizeMode(t *testing.T) { + cases := []struct { + name string + in *string + want string + }{ + {"nil", nil, ""}, + {"empty string", stringPtr(""), ""}, + {"mixed case", stringPtr("File"), "file"}, + {"with spaces", stringPtr(" prompty "), "prompty"}, + {"all caps", stringPtr("AGENTYAML"), "agentyaml"}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, normalizeMode(tt.in)) + }) + } +} + +func stringPtr(s string) *string { return &s } diff --git a/configuration/git/poll.go b/configuration/git/poll.go new file mode 100644 index 0000000000..03ca6888dd --- /dev/null +++ b/configuration/git/poll.go @@ -0,0 +1,260 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "errors" + "time" + + "github.com/go-git/go-git/v5/plumbing" + + "github.com/dapr/components-contrib/configuration" + "github.com/dapr/components-contrib/configuration/git/auth" +) + +// pollLoop ticks every metadata.PollInterval until ctx is cancelled. +// +// When tick reports a rate-limit error (from go-git's HTTP transport or +// from the GitHub App installation-token fetcher), the loop pauses for +// `rateLimitRetryAfter` (or the server-suggested Retry-After when known) +// before its next attempt — protecting the provider's rate-limit budget +// for the rest of the deployment and avoiding hammering the API while it +// is asking us to back off. +func (s *ConfigurationStore) pollLoop(ctx context.Context) { + defer s.pollerWg.Done() + ticker := time.NewTicker(s.metadata.pollInterval()) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + if err := s.tick(ctx); err != nil { + if errors.Is(err, context.Canceled) { + return + } + backoff := s.rateLimitBackoff(err) + if backoff > 0 { + s.logger.Warnf("git config: rate-limited by upstream; backing off for %s before next tick: %v", backoff, err) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + continue + } + s.logger.Warnf("git config: poll tick failed (will retry): %v", err) + } + } +} + +// rateLimitBackoff returns the duration to wait before the next tick when +// err is a rate-limit response. Returns zero if err is not a rate-limit +// signal — the loop then proceeds on its normal cadence. +func (s *ConfigurationStore) rateLimitBackoff(err error) time.Duration { + var rl *auth.RateLimitError + if errors.As(err, &rl) { + if rl.RetryAfter > 0 { + return rl.RetryAfter + } + return s.metadata.rateLimitRetryAfter() + } + if auth.IsTransportRateLimit(err) { + return s.metadata.rateLimitRetryAfter() + } + return 0 +} + +// tick fetches the upstream branch and, if HEAD has moved, walks the tree, +// builds a new snapshot, and fans out diffs to subscribers. Uses the fetch +// itself (via gogit.NoErrAlreadyUpToDate) to detect change rather than a +// separate ls-remote probe — the probe downloads the entire ref advertisement +// and counts as a full request against the provider rate limit, so skipping +// it is a meaningful budget saving on large remotes. +func (s *ConfigurationStore) tick(ctx context.Context) error { + fetchCtx, cancel := context.WithTimeout(ctx, s.metadata.fetchTimeout()) + defer cancel() + head, changed, err := fetchIfChanged(fetchCtx, s.repo, s.metadata.branch(), s.auth) + if err != nil { + return err + } + if !changed { + return nil + } + return s.refreshAndFanOut(head) +} + +// refreshAndFanOut walks the worktree, builds a snapshot, and dispatches +// per-subscriber diffs. Network and disk I/O happen outside the lock; the +// lock is held only for the brief atomic snapshot/registry update plus the +// O(N) collection of subscriber state. Diff computation and handler calls +// happen outside the lock. +func (s *ConfigurationStore) refreshAndFanOut(head plumbing.Hash) error { + entries, err := walkTree(s.workdir, s.metadata.path(), s.metadata.includeHidden(), s.metadata.maxFileSize(), s.logger) + if err != nil { + return err + } + + version := shortSHA(head) + newSnap, err := s.mapper.Map(entries, version, s.logger) + if err != nil { + return err + } + + subs := s.installSnapshot(head, newSnap) + for _, sub := range subs { + base := sub.base + diff := diffSnapshots(base, newSnap, sub.keys, version) + if len(diff) == 0 { + continue + } + event := &configuration.UpdateEvent{ID: sub.id, Items: diff} + if err := sub.handler(sub.ctx, event); err != nil { + s.logger.Errorf("git config: subscriber %s handler error: %v", sub.id, err) + } + } + return nil +} + +// subSnapshot is per-subscriber state captured under the lock for use in +// outside-the-lock diff computation and dispatch. +type subSnapshot struct { + id string + keys []string + handler configuration.UpdateHandler + ctx context.Context + base map[string]*configuration.Item +} + +// installSnapshot atomically replaces the shared snapshot, advances every +// subscriber's deliveredHEAD, captures each subscriber's prior diff base +// from the LRU, and returns a slice of subscriber snapshots for dispatch +// outside the lock. +func (s *ConfigurationStore) installSnapshot(head plumbing.Hash, newSnap map[string]*configuration.Item) []subSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + + s.snapshot = newSnap + s.snapshotHead = head + if s.lru != nil { + s.lru.Add(head, newSnap) + } + + subs := make([]subSnapshot, 0, len(s.subscriptions)) + for _, sub := range s.subscriptions { + subs = append(subs, subSnapshot{ + id: sub.id, + keys: sub.keys, + handler: sub.handler, + ctx: sub.ctx, + base: s.lookupSnapshot(sub.deliveredHEAD), + }) + sub.deliveredHEAD = head + } + return subs +} + +// lookupSnapshot returns the cached snapshot for hash, or nil if not found. +// Caller must hold s.mu. +func (s *ConfigurationStore) lookupSnapshot(hash plumbing.Hash) map[string]*configuration.Item { + if s.lru == nil || hash.IsZero() { + return nil + } + if snap, ok := s.lru.Get(hash); ok { + return snap + } + return nil +} + +// diffSnapshots returns items changed between prev and next, optionally +// filtered by keys. Deletions surface as Item{Value:"", Metadata:{deleted:true}}. +// version is applied to every emitted item. +func diffSnapshots(prev, next map[string]*configuration.Item, keys []string, version string) map[string]*configuration.Item { + keyFilter := keysAsSet(keys) + out := make(map[string]*configuration.Item) + + for k, nv := range next { + if !keyFilter.match(k) { + continue + } + pv, hadPrev := prev[k] + if !hadPrev || pv == nil || pv.Value != nv.Value { + out[k] = &configuration.Item{ + Value: nv.Value, + Version: version, + Metadata: cloneMetadata(nv.Metadata), + } + } + } + for k := range prev { + if !keyFilter.match(k) { + continue + } + if _, ok := next[k]; ok { + continue + } + out[k] = &configuration.Item{ + Value: "", + Version: version, + Metadata: map[string]string{"deleted": "true"}, + } + } + return out +} + +// cloneMetadata returns a freshly-allocated copy of the input metadata. +// Every emitted Item gets its own metadata map so handlers can safely +// mutate the map they receive without corrupting other events or the +// store's internal state. +func cloneMetadata(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +type keySet struct { + all bool + want map[string]struct{} +} + +func (k keySet) match(s string) bool { + if k.all { + return true + } + _, ok := k.want[s] + return ok +} + +func keysAsSet(keys []string) keySet { + if len(keys) == 0 { + return keySet{all: true} + } + want := make(map[string]struct{}, len(keys)) + for _, k := range keys { + want[k] = struct{}{} + } + return keySet{want: want} +} + +func shortSHA(h plumbing.Hash) string { + s := h.String() + if len(s) > 7 { + return s[:7] + } + return s +} diff --git a/configuration/git/poll_test.go b/configuration/git/poll_test.go new file mode 100644 index 0000000000..927bb9ba66 --- /dev/null +++ b/configuration/git/poll_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/dapr/components-contrib/configuration" +) + +func TestDiffSnapshots(t *testing.T) { + prev := map[string]*configuration.Item{ + "a": {Value: "1"}, + "b": {Value: "2"}, + "c": {Value: "3"}, + } + next := map[string]*configuration.Item{ + "a": {Value: "1"}, + "b": {Value: "2-changed"}, + "d": {Value: "4"}, + } + got := diffSnapshots(prev, next, nil, "abc1234") + + assert.Len(t, got, 3) + + // Changed + assert.Equal(t, "2-changed", got["b"].Value) + assert.Equal(t, "abc1234", got["b"].Version) + + // Added + assert.Equal(t, "4", got["d"].Value) + + // Deleted + assert.Equal(t, "", got["c"].Value) + assert.Equal(t, "true", got["c"].Metadata["deleted"]) + + // Unchanged + _, ok := got["a"] + assert.False(t, ok) +} + +func TestDiffSnapshots_KeyFilter(t *testing.T) { + prev := map[string]*configuration.Item{ + "alpha": {Value: "1"}, + "beta": {Value: "2"}, + } + next := map[string]*configuration.Item{ + "alpha": {Value: "1-changed"}, + "beta": {Value: "2-changed"}, + } + got := diffSnapshots(prev, next, []string{"alpha"}, "v1") + assert.Len(t, got, 1) + assert.Equal(t, "1-changed", got["alpha"].Value) +} + +func TestDiffSnapshots_LRUMiss(t *testing.T) { + // LRU miss → prev=nil → everything in `next` is emitted as add. + next := map[string]*configuration.Item{ + "key1": {Value: "v1"}, + "key2": {Value: "v2"}, + } + got := diffSnapshots(nil, next, nil, "v1") + assert.Len(t, got, 2) + assert.Equal(t, "v1", got["key1"].Value) + assert.Equal(t, "v2", got["key2"].Value) +} + +func TestKeySet(t *testing.T) { + all := keysAsSet(nil) + assert.True(t, all.match("anything")) + + some := keysAsSet([]string{"x", "y"}) + assert.True(t, some.match("x")) + assert.False(t, some.match("z")) +} diff --git a/configuration/git/subscriber.go b/configuration/git/subscriber.go new file mode 100644 index 0000000000..fb4f238102 --- /dev/null +++ b/configuration/git/subscriber.go @@ -0,0 +1,80 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + + "github.com/go-git/go-git/v5/plumbing" + + "github.com/dapr/components-contrib/configuration" +) + +// subscription holds the per-subscriber state held by the store. +// +// `ctx` is the subscriber's child context — cancelled when the subscriber +// calls Unsubscribe. Handlers receive this context so they can detect their +// own subscription being torn down (e.g. to abort a downstream gRPC stream). +type subscription struct { + id string + keys []string + handler configuration.UpdateHandler + deliveredHEAD plumbing.Hash + ctx context.Context + cancel context.CancelFunc +} + +// cloneItem returns a deep copy of in — fresh Value, Version, and Metadata +// map. The Metadata is allocated even when the source is empty so callers +// can mutate it safely. +func cloneItem(in *configuration.Item) *configuration.Item { + if in == nil { + return nil + } + out := &configuration.Item{ + Value: in.Value, + Version: in.Version, + } + if len(in.Metadata) == 0 { + out.Metadata = map[string]string{} + return out + } + out.Metadata = make(map[string]string, len(in.Metadata)) + for k, v := range in.Metadata { + out.Metadata[k] = v + } + return out +} + +// filterByKeys returns deep-cloned items containing only the requested keys. +// Callers own the returned items and may freely mutate them — the store's +// internal snapshot is never aliased. +// +// An empty keys slice means "all keys". +func filterByKeys(items map[string]*configuration.Item, keys []string) map[string]*configuration.Item { + if len(keys) == 0 { + out := make(map[string]*configuration.Item, len(items)) + for k, v := range items { + out[k] = cloneItem(v) + } + return out + } + out := make(map[string]*configuration.Item, len(keys)) + for _, k := range keys { + if v, ok := items[k]; ok { + out[k] = cloneItem(v) + } + } + return out +} diff --git a/configuration/git/walk.go b/configuration/git/walk.go new file mode 100644 index 0000000000..b846c7cfff --- /dev/null +++ b/configuration/git/walk.go @@ -0,0 +1,118 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/dapr/kit/logger" +) + +// walkTree walks files under root/sub and returns POSIX-pathed entries +// relative to root/sub. Symlinks are skipped. Hidden files (basename +// starts with `.`) are skipped unless includeHidden is true. The `.git` +// directory is **always** excluded regardless of includeHidden so that +// the remote URL stored in `.git/config` (and any embedded credentials) +// cannot leak into the configuration store output. Files larger than +// maxFileSize are skipped with a warning to keep memory bounded. +func walkTree(root, sub string, includeHidden bool, maxFileSize int64, log logger.Logger) ([]fileEntry, error) { + if root == "" { + return nil, errors.New("walkTree: empty root") + } + scope := filepath.Join(root, filepath.FromSlash(sub)) + info, err := os.Stat(scope) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("stat %q: %w", scope, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("path %q is not a directory", scope) + } + + var entries []fileEntry + err = filepath.WalkDir(scope, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // `.git` is always excluded regardless of type (directory, file, or + // other) and regardless of includeHidden. Submodules and linked + // worktrees use a `.git` *file* containing `gitdir: `, which + // could leak through a directory-only check. Returning fs.SkipDir + // from a non-directory entry is treated as "stop descending the + // current parent" — for a leaf file we want to skip the file but + // keep walking its siblings, so we return nil in that case. + if d.Name() == ".git" { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + // Skip hidden files/dirs unless includeHidden is set. + if !includeHidden && strings.HasPrefix(d.Name(), ".") && p != scope { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + if d.IsDir() { + return nil + } + // Always call Info() so we can rely on its mode for both the symlink + // check and the size cap. fs.DirEntry.Type() returns a partial mode + // that on some platforms may not include the symlink bit unless an + // lstat has been performed; Info() guarantees it. + fi, statErr := d.Info() + if statErr != nil { + return fmt.Errorf("stat %q: %w", p, statErr) + } + // Skip symlinks — never follow. Checking fi.Mode() (which is the + // result of an lstat on the entry) avoids the partial-mode pitfall + // of d.Type(), which could otherwise let a symlink slip through and + // be read via os.ReadFile (which follows links). + if fi.Mode()&os.ModeSymlink != 0 { + return nil + } + rel, err := filepath.Rel(scope, p) + if err != nil { + return fmt.Errorf("rel %q: %w", p, err) + } + if maxFileSize > 0 && fi.Size() > maxFileSize { + if log != nil { + log.Warnf("git config: skipping %q (size %d exceeds maxFileSize %d)", + filepath.ToSlash(rel), fi.Size(), maxFileSize) + } + return nil + } + data, err := os.ReadFile(p) + if err != nil { + return fmt.Errorf("read %q: %w", p, err) + } + entries = append(entries, fileEntry{ + RelPath: filepath.ToSlash(rel), + Bytes: data, + }) + return nil + }) + if err != nil { + return nil, err + } + return entries, nil +} diff --git a/configuration/git/walk_test.go b/configuration/git/walk_test.go new file mode 100644 index 0000000000..9576d8ad99 --- /dev/null +++ b/configuration/git/walk_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalkTree_NonexistentScopeReturnsEmpty(t *testing.T) { + root := t.TempDir() + entries, err := walkTree(root, "nope", false, 0, nil) + require.NoError(t, err) + assert.Nil(t, entries) +} + +func TestWalkTree_FileScopeReturnsError(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "scope"), []byte("file not dir"), 0o600)) + _, err := walkTree(root, "scope", false, 0, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a directory") +} + +func TestWalkTree_HiddenSkipDefault(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "visible.txt"), []byte("v"), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(root, ".hidden"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".hidden", "ignored.txt"), []byte("x"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".dotfile"), []byte("x"), 0o600)) + + entries, err := walkTree(root, ".", false, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "visible.txt", entries[0].RelPath) +} + +func TestWalkTree_GitDirAlwaysExcluded(t *testing.T) { + // .git/config typically contains the remote URL and may embed credentials. + // It must be skipped even with includeHidden=true. + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".git", "config"), + []byte("[remote \"origin\"]\n url = https://user:tok@example.com/repo\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "k.txt"), []byte("v"), 0o600)) + + entries, err := walkTree(root, ".", true, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "k.txt", entries[0].RelPath) +} + +func TestWalkTree_GitAsScopeReturnsEmpty(t *testing.T) { + // Defence-in-depth backstop: even if a malformed config bypassed + // metadata validation and pointed `path` at `.git`, the walker must + // refuse to enter it and produce no entries. + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".git", "config"), + []byte("[remote \"origin\"]\n url = https://user:tok@example.com/repo\n"), 0o600)) + + entries, err := walkTree(root, ".git", true, 0, nil) + require.NoError(t, err) + assert.Empty(t, entries, "walker must not surface contents of a .git scope") +} + +func TestWalkTree_GitInsideSubdirAlsoExcluded(t *testing.T) { + // A nested .git directory must also be skipped — defends against + // submodule-like layouts or stray .git checkins inside a config repo. + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "agents", ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "agents", ".git", "config"), + []byte("secret"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "agents", "k.txt"), []byte("v"), 0o600)) + + entries, err := walkTree(root, "agents", true, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "k.txt", entries[0].RelPath) +} + +func TestWalkTree_GitFileExcluded(t *testing.T) { + // Submodules and linked worktrees use a `.git` *file* containing + // `gitdir: `. A directory-only exclusion would leak it. + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, ".git"), + []byte("gitdir: ../somewhere-else/.git\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "k.txt"), []byte("v"), 0o600)) + + entries, err := walkTree(root, ".", true, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "k.txt", entries[0].RelPath) +} + +func TestWalkTree_SymlinksSkipped(t *testing.T) { + // Symlinks must not be followed — they could otherwise read files + // outside the configured scope. Uses Info().Mode() because some + // platforms don't populate the symlink bit in fs.DirEntry.Type() + // without an lstat. + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "target.txt"), + []byte("real"), 0o600)) + if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil { + t.Skipf("symlink not supported on this platform: %v", err) + } + + entries, err := walkTree(root, ".", true, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1, "symlink should be skipped") + assert.Equal(t, "target.txt", entries[0].RelPath) +} + +func TestWalkTree_FileSizeCap(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "small.txt"), []byte("ok"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "big.txt"), make([]byte, 4096), 0o600)) + + entries, err := walkTree(root, ".", false, 1024, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "small.txt", entries[0].RelPath) +} + +func TestWalkTree_IncludeHidden(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "v.txt"), []byte("v"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".dotfile"), []byte("x"), 0o600)) + + entries, err := walkTree(root, ".", true, 0, nil) + require.NoError(t, err) + assert.Len(t, entries, 2) +} + +func TestWalkTree_NestedAndPosixPaths(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "agents", "weather"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "agents", "weather", "role.txt"), []byte("r"), 0o600)) + + entries, err := walkTree(root, "agents", false, 0, nil) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "weather/role.txt", entries[0].RelPath) +} diff --git a/go.mod b/go.mod index 171cafc93a..0302e8f4ae 100644 --- a/go.mod +++ b/go.mod @@ -74,6 +74,7 @@ require ( github.com/didip/tollbooth/v7 v7.0.1 github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/fasthttp-contrib/sessions v0.0.0-20160905201309-74f6ac73d5d5 + github.com/go-git/go-git/v5 v5.18.0 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.8.1 github.com/go-zookeeper/zk v1.0.3 @@ -181,6 +182,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/RoaringBitmap/roaring v1.1.0 // indirect github.com/RoaringBitmap/roaring/v2 v2.8.0 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -227,6 +229,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj/v2 v2.5.6 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cloudwego/configmanager v0.2.3 // indirect github.com/cloudwego/dynamicgo v0.8.0 // indirect @@ -272,6 +275,8 @@ require ( github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gage-technologies/mistral-go v1.1.0 // indirect github.com/gavv/httpexpect v2.0.0+incompatible // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.8.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-kit/kit v0.10.0 // indirect @@ -327,6 +332,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -338,6 +344,7 @@ require ( github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/kataras/go-errors v0.0.3 // indirect github.com/kataras/go-serializer v0.0.4 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/knadh/koanf v1.4.1 // indirect @@ -387,6 +394,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect @@ -405,10 +413,12 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shirou/gopsutil/v4 v4.25.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -425,6 +435,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -473,6 +484,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/kataras/go-serializer.v0 v0.0.4 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect diff --git a/go.sum b/go.sum index 1eb871b203..2898f3d672 100644 --- a/go.sum +++ b/go.sum @@ -144,11 +144,14 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY github.com/IBM/sarama v1.48.1 h1:x1dSWebprjjE7Wr7n8RVAxwa4mt4O9JejRxnZrGIXk0= github.com/IBM/sarama v1.48.1/go.mod h1:m/Q1aFezH82/AglfTpJbw/fO0ZybYXhPgTmvajiZX50= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d h1:wvStE9wLpws31NiWUx+38wny1msZ/tm+eL5xmm4Y7So= github.com/Netflix/go-env v0.0.0-20220526054621-78278af1949d/go.mod h1:9XMFaCeRyW7fC9XJOWQ+NdAv8VLG7ys7l3x4ozEGLUQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/RoaringBitmap/roaring v1.1.0 h1:b10lZrZXaY6Q6EKIRrmOF519FIyQQ5anPgGr3niw2yY= github.com/RoaringBitmap/roaring v1.1.0/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring/v2 v2.8.0 h1:y1rdtixfXvaITKzkfiKvScI0hlBJHe9sfzJp8cgeM7w= @@ -223,6 +226,8 @@ github.com/aliyunmq/mq-http-go-sdk v1.0.3/go.mod h1:JYfRMQoPexERvnNNBcal0ZQ2TVQ5 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/dubbo-getty v1.4.9-0.20220610060150-8af010f3f3dc h1:NZRon3MDqT4vddR3UIRBnwbbhEerghAimCSBsiESs3g= github.com/apache/dubbo-getty v1.4.9-0.20220610060150-8af010f3f3dc/go.mod h1:cPJlbcHUTNTpiboMQjMHhE9XBni11LiBiG8FdrDuVzk= @@ -252,6 +257,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -413,6 +420,8 @@ github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= @@ -565,6 +574,8 @@ github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2I github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -636,10 +647,20 @@ github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49P github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-co-op/gocron v1.9.0/go.mod h1:DbJm9kdgr1sEvWpHCA7dFFs/PGHPMil9/97EXCRPr4k= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= +github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.18.0 h1:O831KI+0PR51hM2kep6T8k+w0/LIAD490gvqMCvL5hM= +github.com/go-git/go-git/v5 v5.18.0/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -1020,6 +1041,8 @@ github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg= github.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -1074,6 +1097,8 @@ github.com/kataras/go-errors v0.0.3 h1:RQSGEb5AHjsGbwhNW8mFC7a9JrgoCLHC8CBQ4keXJ github.com/kataras/go-errors v0.0.3/go.mod h1:K3ncz8UzwI3bpuksXt5tQLmrRlgxfv+52ARvAu1+I+o= github.com/kataras/go-serializer v0.0.4 h1:isugggrY3DSac67duzQ/tn31mGAUtYqNpE2ob6Xt/SY= github.com/kataras/go-serializer v0.0.4/go.mod h1:/EyLBhXKQOJ12dZwpUZZje3lGy+3wnvG7QKaVJtm/no= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1380,6 +1405,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -1506,8 +1533,8 @@ github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekuei github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= github.com/sendgrid/sendgrid-go v3.13.0+incompatible h1:HZrzc06/QfBGesY9o3n1lvBrRONA+57rbDRKet7plos= github.com/sendgrid/sendgrid-go v3.13.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= @@ -1531,6 +1558,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1673,6 +1702,8 @@ github.com/vmware/vmware-go-kcl-v2 v1.0.0 h1:HPT5vu+khRmGspBSc/+AilEWbRGoTZhjlYq github.com/vmware/vmware-go-kcl-v2 v1.0.0/go.mod h1:GBDu+P4Neo0vwZAk0ZUCEC8GYsUOWvi3XhFwAZR3SjA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -2083,6 +2114,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2401,6 +2433,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/tests/certification/configuration/git/components/default/configstore.yaml b/tests/certification/configuration/git/components/default/configstore.yaml new file mode 100644 index 0000000000..d35e39b15d --- /dev/null +++ b/tests/certification/configuration/git/components/default/configstore.yaml @@ -0,0 +1,15 @@ +# Placeholder — the certification test writes a fresh spec to a temp dir at +# runtime with the upstream URL injected (see writeComponentSpec in +# git_test.go). This file exists only so the directory structure is present +# for tooling that walks tests/certification/. Init against this spec will +# fail intentionally because the URL points at a non-existent path. +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: configstore +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "file:///__cert_test_placeholder_replaced_at_runtime__" diff --git a/tests/certification/configuration/git/config.yaml b/tests/certification/configuration/git/config.yaml new file mode 100644 index 0000000000..40c2c68759 --- /dev/null +++ b/tests/certification/configuration/git/config.yaml @@ -0,0 +1,5 @@ +apiVersion: dapr.io/v1alpha1 +kind: Configuration +metadata: + name: daprconfig +spec: {} diff --git a/tests/certification/configuration/git/git_test.go b/tests/certification/configuration/git/git_test.go new file mode 100644 index 0000000000..f869f6f6e5 --- /dev/null +++ b/tests/certification/configuration/git/git_test.go @@ -0,0 +1,349 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git_test + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dapr/components-contrib/configuration" + config_git "github.com/dapr/components-contrib/configuration/git" + "github.com/dapr/components-contrib/tests/certification/embedded" + "github.com/dapr/components-contrib/tests/certification/flow" + "github.com/dapr/components-contrib/tests/certification/flow/sidecar" + "github.com/dapr/components-contrib/tests/certification/flow/watcher" + cu_git "github.com/dapr/components-contrib/tests/utils/configupdater/git" + configuration_loader "github.com/dapr/dapr/pkg/components/configuration" + dapr_testing "github.com/dapr/dapr/pkg/testing" + dapr "github.com/dapr/go-sdk/client" + "github.com/dapr/kit/logger" +) + +const ( + storeName = "configstore" + key1 = "key1" + key2 = "key2" + val1 = "val1" + val2 = "val2" + sidecarName1 = "dapr-1" + pollInterval = "250ms" +) + +// castConfigurationItems converts go-sdk ConfigurationItem to contrib ConfigurationItem. +func castConfigurationItems(items map[string]*dapr.ConfigurationItem) map[string]*configuration.Item { + configItems := make(map[string]*configuration.Item) + for key, item := range items { + configItems[key] = &configuration.Item{ + Value: item.Value, + Version: item.Version, + Metadata: item.Metadata, + } + } + return configItems +} + +// seedUpstream initialises a bare git upstream and returns its file:// URL. +// Cleanup is registered with t.Cleanup. The upstream starts with a single +// empty commit on `main` so the store-under-test can clone successfully. +func seedUpstream(t *testing.T) string { + t.Helper() + root, err := os.MkdirTemp("", "dapr-cert-git-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(root) }) + + bare := filepath.Join(root, "upstream.git") + seed := filepath.Join(root, "seed") + + mustRun := func(dir, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "%s %v: %s", name, args, out) + } + + mustRun("", "git", "init", "--bare", bare) + mustRun("", "git", "clone", bare, seed) + mustRun(seed, "git", "config", "user.email", "cert@dapr.io") + mustRun(seed, "git", "config", "user.name", "cert") + mustRun(seed, "git", "config", "commit.gpgsign", "false") + mustRun(seed, "git", "commit", "--allow-empty", "-m", "initial") + mustRun(seed, "git", "branch", "-M", "main") + mustRun(seed, "git", "push", "origin", "main") + // Point the bare repo's HEAD at refs/heads/main so go-git's PlainClone + // in the updater can resolve a meaningful default branch instead of + // falling into the no-refs path. + mustRun("", "git", "--git-dir", bare, "symbolic-ref", "HEAD", "refs/heads/main") + + return fileURL(bare) +} + +// fileURL produces a portable file:// URL for an absolute local path. +// On Unix the path already starts with `/`, so `file://` + path yields the +// canonical three-slash form. On Windows the path starts with a drive +// letter (e.g. `C:\foo`), so we normalise separators and prepend `/` +// to produce the standard `file:///C:/foo` form. +func fileURL(absPath string) string { + p := filepath.ToSlash(absPath) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return "file://" + p +} + +// writeComponentSpec writes the Dapr component spec for the configuration +// store under test to a fresh temp directory and returns its path. The +// upstream URL is injected directly because Dapr does not interpolate +// environment variables in component metadata values. Using a temp dir +// avoids overwriting the placeholder spec checked into components/default. +func writeComponentSpec(t *testing.T, upstreamURL string) string { + t.Helper() + dir, err := os.MkdirTemp("", "dapr-cert-git-components-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + spec := fmt.Sprintf(`apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: configstore +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: %q + - name: branch + value: "main" + - name: path + value: "." + - name: mappingMode + value: "file" + - name: pollInterval + value: %q + - name: emitInitialState + value: "false" +`, upstreamURL, pollInterval) + path := filepath.Join(dir, "configstore.yaml") + require.NoError(t, os.WriteFile(path, []byte(spec), 0o644)) + return dir +} + +func TestGit(t *testing.T) { + log := logger.NewLogger("dapr.components") + + upstreamURL := seedUpstream(t) + componentsPath := writeComponentSpec(t, upstreamURL) + + configurationRegistry := configuration_loader.NewRegistry() + configurationRegistry.Logger = log + configurationRegistry.RegisterComponent(config_git.NewGitConfigurationStore, "git") + + updater := cu_git.NewGitConfigUpdater(log).(*cu_git.ConfigUpdater) + + ports, err := dapr_testing.GetFreePorts(2) + require.NoError(t, err) + currentGrpcPort := ports[0] + currentHTTPPort := ports[1] + + messageWatcher := watcher.NewUnordered() + var subscribeIDs []string + + initUpdater := func(_ flow.Context) error { + return updater.Init(map[string]string{ + "remoteUrl": upstreamURL, + "branch": "main", + "pollInterval": pollInterval, + }) + } + + checkConnection := func(_ flow.Context) error { + probe := map[string]*configuration.Item{"healthcheck": {Value: "ok"}} + if err := updater.AddKey(probe); err != nil { + return err + } + return updater.DeleteKey([]string{"healthcheck"}) + } + + testGet := func(ctx flow.Context) error { + client := sidecar.GetClient(ctx, sidecarName1) + + require.NoError(ctx.T, updater.AddKey(map[string]*configuration.Item{ + key1: {Value: val1}, + key2: {Value: val2}, + })) + + // Wait until the sidecar's snapshot reflects the writes — uses + // require.Eventually rather than a fixed-interval sleep loop. + var items map[string]*dapr.ConfigurationItem + require.Eventually(ctx.T, func() bool { + items, err = client.GetConfigurationItems(ctx, storeName, []string{key1}) + return err == nil && len(items) == 1 + }, 15*time.Second, 200*time.Millisecond, "snapshot did not reflect write within deadline") + require.NoError(ctx.T, err) + require.Len(ctx.T, items, 1) + require.Equal(ctx.T, val1, items[key1].Value) + + items, err = client.GetConfigurationItems(ctx, storeName, []string{key1, key2}) + require.NoError(ctx.T, err) + require.Len(ctx.T, items, 2) + require.Equal(ctx.T, val1, items[key1].Value) + require.Equal(ctx.T, val2, items[key2].Value) + + items, err = client.GetConfigurationItems(ctx, storeName, []string{"nonexistent"}) + require.NoError(ctx.T, err) + require.Empty(ctx.T, items) + return nil + } + + subscribeFn := func(keys []string, message *watcher.Watcher) flow.Runnable { + return func(ctx flow.Context) error { + client := sidecar.GetClient(ctx, sidecarName1) + message.Reset() + subID, errSubscribe := client.SubscribeConfigurationItems(ctx, storeName, keys, + func(_ string, items map[string]*dapr.ConfigurationItem) { + evt := &configuration.UpdateEvent{Items: castConfigurationItems(items)} + // Strip system-assigned version (commit SHA) and normalise nil + // metadata for stable JSON comparison. + for _, item := range evt.Items { + item.Version = "" + if item.Metadata == nil { + item.Metadata = map[string]string{} + } + } + payload, err := json.Marshal(evt) + if err != nil { + log.Errorf("failed to marshal subscription event: %v", err) + return + } + message.Observe(string(payload)) + }) + subscribeIDs = append(subscribeIDs, subID) + return errSubscribe + } + } + + verifySubscriberReady := func(messages *watcher.Watcher) flow.Runnable { + return func(ctx flow.Context) error { + messages.Reset() + probe := configuration.UpdateEvent{ + Items: map[string]*configuration.Item{ + key1: {Value: "readiness-probe", Metadata: map[string]string{}}, + }, + } + payload, err := json.Marshal(probe) + if err != nil { + return fmt.Errorf("marshal probe: %w", err) + } + messages.Expect(string(payload)) + if err := updater.UpdateKey(map[string]*configuration.Item{ + key1: {Value: "readiness-probe"}, + }); err != nil { + return fmt.Errorf("update probe: %w", err) + } + messages.Assert(ctx.T, 30*time.Second) + return nil + } + } + + testSubscribe := func(messages *watcher.Watcher) flow.Runnable { + return func(ctx flow.Context) error { + messages.Reset() + expected := configuration.UpdateEvent{ + Items: map[string]*configuration.Item{ + key1: {Value: "updated-val1", Metadata: map[string]string{}}, + }, + } + payload, err := json.Marshal(expected) + if err != nil { + return fmt.Errorf("marshal expected update: %w", err) + } + messages.Expect(string(payload)) + require.NoError(ctx.T, updater.UpdateKey(map[string]*configuration.Item{ + key1: {Value: "updated-val1"}, + })) + messages.Assert(ctx.T, 30*time.Second) + return nil + } + } + + testDelete := func(messages *watcher.Watcher) flow.Runnable { + return func(ctx flow.Context) error { + messages.Reset() + expected := configuration.UpdateEvent{ + Items: map[string]*configuration.Item{ + key2: {Value: "", Metadata: map[string]string{"deleted": "true"}}, + }, + } + payload, err := json.Marshal(expected) + if err != nil { + return fmt.Errorf("marshal expected delete: %w", err) + } + messages.Expect(string(payload)) + require.NoError(ctx.T, updater.DeleteKey([]string{key2})) + messages.Assert(ctx.T, 30*time.Second) + return nil + } + } + + stopSubscribers := func(ctx flow.Context) error { + client := sidecar.GetClient(ctx, sidecarName1) + for _, subID := range subscribeIDs { + if err := client.UnsubscribeConfigurationItems(ctx, storeName, subID); err != nil { + return err + } + } + subscribeIDs = nil + return nil + } + + testGetAfterUpdate := func(ctx flow.Context) error { + client := sidecar.GetClient(ctx, sidecarName1) + items, err := client.GetConfigurationItems(ctx, storeName, []string{key1}) + require.NoError(ctx.T, err) + require.Len(ctx.T, items, 1) + require.Equal(ctx.T, "updated-val1", items[key1].Value) + return nil + } + + flow.New(t, "git configuration certification test"). + Step("initialize updater", initUpdater). + Step("verify connection", checkConnection). + Step(sidecar.Run(sidecarName1, + embedded.WithoutApp(), + embedded.WithDaprGRPCPort(strconv.Itoa(currentGrpcPort)), + embedded.WithDaprHTTPPort(strconv.Itoa(currentHTTPPort)), + embedded.WithResourcesPath(componentsPath), + embedded.WithConfigurations(configurationRegistry), + )). + Step("test get", testGet). + Step("start subscriber", subscribeFn([]string{key1, key2}, messageWatcher)). + Step("verify subscriber ready", verifySubscriberReady(messageWatcher)). + Step("reset", flow.Reset(messageWatcher)). + Step("test subscribe updates", testSubscribe(messageWatcher)). + Step("reset", flow.Reset(messageWatcher)). + Step("test delete notifications", testDelete(messageWatcher)). + Step("reset", flow.Reset(messageWatcher)). + Step("stop subscribers", stopSubscribers). + Step("verify data after unsubscribe", testGetAfterUpdate). + Run() +} diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 8eac6b8270..e7e0e5cf3d 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -55,6 +55,7 @@ require ( cloud.google.com/go/datastore v1.20.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect + dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/AthenZ/athenz v1.12.13 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect @@ -75,6 +76,7 @@ require ( github.com/Code-Hex/go-generics-cache v1.3.1 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/RoaringBitmap/roaring v1.1.0 // indirect github.com/RoaringBitmap/roaring/v2 v2.8.0 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -128,6 +130,7 @@ require ( github.com/chebyrash/promise v0.0.0-20230709133807-42ec49ba1459 // indirect github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 // indirect github.com/cloudevents/sdk-go/v2 v2.15.2 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cloudwego/configmanager v0.2.3 // indirect github.com/cloudwego/dynamicgo v0.8.0 // indirect @@ -162,6 +165,9 @@ require ( github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gage-technologies/mistral-go v1.1.0 // indirect github.com/go-chi/cors v1.2.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-git/go-git/v5 v5.18.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -221,6 +227,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -231,6 +238,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/knadh/koanf v1.4.1 // indirect @@ -270,6 +278,7 @@ require ( github.com/panjf2000/ants/v2 v2.11.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect @@ -288,9 +297,11 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.5.4 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/smartystreets/assertions v1.1.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -312,6 +323,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/vmware/vmware-go-kcl-v2 v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -358,6 +370,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.32.3 // indirect diff --git a/tests/certification/go.sum b/tests/certification/go.sum index c077f1833e..3a1b18baac 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -116,9 +116,12 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY github.com/IBM/sarama v1.48.1 h1:x1dSWebprjjE7Wr7n8RVAxwa4mt4O9JejRxnZrGIXk0= github.com/IBM/sarama v1.48.1/go.mod h1:m/Q1aFezH82/AglfTpJbw/fO0ZybYXhPgTmvajiZX50= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/RoaringBitmap/roaring v1.1.0 h1:b10lZrZXaY6Q6EKIRrmOF519FIyQQ5anPgGr3niw2yY= github.com/RoaringBitmap/roaring v1.1.0/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring/v2 v2.8.0 h1:y1rdtixfXvaITKzkfiKvScI0hlBJHe9sfzJp8cgeM7w= @@ -154,6 +157,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.18/go.mod h1:v8ESoHo4SyHmuB4b1tJqDH github.com/aliyun/alibaba-cloud-sdk-go v1.61.1704/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/alphadose/haxmap v1.4.0 h1:1yn+oGzy2THJj1DMuJBzRanE3sMnDAjJVbU0L31Jp3w= github.com/alphadose/haxmap v1.4.0/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/anshal21/go-worker v1.1.0 h1:TPt2jBN/6dmPDPDTq8DHA0MtoXG8RWKGoJVHqED+s5g= github.com/anshal21/go-worker v1.1.0/go.mod h1:6GiLOIr/VvVg80vfW65ytLuouSvndU2IoJTu+8M47lI= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -181,6 +186,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -326,6 +333,8 @@ github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2 h1:FIvfKlS2mcuP github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.15.2/go.mod h1:POsdVp/08Mki0WD9QvvgRRpg9CQ6zhjfRrBoEY8JFS8= github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cloudwego/configmanager v0.2.3 h1:P0YTBgqDBnKeI/VARvut/Dc9Rfxt9Bw1Nv7sk0Ru4u8= @@ -471,6 +480,8 @@ github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2I github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -531,6 +542,8 @@ github.com/gage-technologies/mistral-go v1.1.0 h1:POv1wM9jA/9OBXGV2YdPi9Y/h09+Mj github.com/gage-technologies/mistral-go v1.1.0/go.mod h1:tF++Xt7U975GcLlzhrjSQb8l/x+PrriO9QEdsgm9l28= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= @@ -538,6 +551,14 @@ github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-co-op/gocron v1.9.0/go.mod h1:DbJm9kdgr1sEvWpHCA7dFFs/PGHPMil9/97EXCRPr4k= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= +github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.18.0 h1:O831KI+0PR51hM2kep6T8k+w0/LIAD490gvqMCvL5hM= +github.com/go-git/go-git/v5 v5.18.0/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -877,6 +898,8 @@ github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg= github.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -927,6 +950,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1153,6 +1178,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -1270,6 +1297,8 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto= github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= @@ -1289,6 +1318,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1408,6 +1439,8 @@ github.com/vmware/vmware-go-kcl-v2 v1.0.0 h1:HPT5vu+khRmGspBSc/+AilEWbRGoTZhjlYq github.com/vmware/vmware-go-kcl-v2 v1.0.0/go.mod h1:GBDu+P4Neo0vwZAk0ZUCEC8GYsUOWvi3XhFwAZR3SjA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -1551,6 +1584,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= @@ -1769,6 +1803,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2036,6 +2071,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/tests/config/configuration/git/local/configstore.yml b/tests/config/configuration/git/local/configstore.yml new file mode 100644 index 0000000000..97206fd453 --- /dev/null +++ b/tests/config/configuration/git/local/configstore.yml @@ -0,0 +1,20 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: configstore +spec: + type: configuration.git + version: v1 + metadata: + - name: remoteUrl + value: "${{GIT_REMOTE_URL}}" + - name: branch + value: "main" + - name: path + value: "." + - name: mappingMode + value: "file" + - name: pollInterval + value: "250ms" # Allowed because the conformance harness uses file:// URLs + - name: emitInitialState + value: "false" diff --git a/tests/config/configuration/tests.yml b/tests/config/configuration/tests.yml index b718c30dcf..526b89258f 100644 --- a/tests/config/configuration/tests.yml +++ b/tests/config/configuration/tests.yml @@ -11,3 +11,5 @@ components: operations: [] - component: kubernetes.kind operations: [] + - component: git.local + operations: [] diff --git a/tests/conformance/configuration/configuration.go b/tests/conformance/configuration/configuration.go index 500294928f..8b6371d1c3 100644 --- a/tests/conformance/configuration/configuration.go +++ b/tests/conformance/configuration/configuration.go @@ -39,16 +39,38 @@ const ( defaultWaitDuration = 5 * time.Second postgresComponent = "postgresql" kubernetesComponent = "kubernetes" + gitComponent = "git" pgNotifyChannelKey = "pgNotifyChannel" pgNotifyChannel = "config" ) +// DeletedItemShape describes how a configuration store represents a deletion +// in an UpdateEvent. +type DeletedItemShape int + +const ( + // DeletedShapeEmptyItem: the receiver expects `Item{Value: "", Metadata: nil-or-empty}`. + // This is the default — used by Redis-style stores that just clear the value. + DeletedShapeEmptyItem DeletedItemShape = iota + // DeletedShapeMetadataFlag: the receiver expects `Item{Value: "", Metadata: {"deleted": "true"}}`. + // Used by stores like Kubernetes ConfigMap and Git that emit a sentinel + // in the metadata so subscribers can distinguish "removed" from "set to empty". + DeletedShapeMetadataFlag + // DeletedShapeUnchanged: the deletion preserves the input items as-is. + // Used by Postgres which surfaces deletions through a different mechanism. + DeletedShapeUnchanged +) + type TestConfig struct { utils.CommonConfig // IgnoreVersion indicates the component uses system-assigned versions - // (e.g. Kubernetes ConfigMap resourceVersion) rather than user-defined versions. - // When true, conformance tests compare items by value and metadata only. + // (e.g. Kubernetes ConfigMap resourceVersion, Git commit SHA) rather than + // user-defined versions. When true, conformance tests compare items by + // value and metadata only. IgnoreVersion bool + // DeletedShape selects how the component signals deletions in an + // UpdateEvent so the harness can build the right expected items. + DeletedShape DeletedItemShape } func NewTestConfig(componentName string, operations []string, configMap map[string]interface{}) TestConfig { @@ -58,12 +80,32 @@ func NewTestConfig(componentName string, operations []string, configMap map[stri ComponentName: componentName, Operations: utils.NewStringSet(operations...), }, - IgnoreVersion: strings.HasPrefix(componentName, kubernetesComponent), + IgnoreVersion: usesSystemAssignedVersion(componentName), + DeletedShape: deletedShapeFor(componentName), } return tc } +func usesSystemAssignedVersion(componentName string) bool { + return strings.HasPrefix(componentName, kubernetesComponent) || + componentName == gitComponent || + strings.HasPrefix(componentName, gitComponent+".") +} + +func deletedShapeFor(componentName string) DeletedItemShape { + switch { + case strings.HasPrefix(componentName, kubernetesComponent), + componentName == gitComponent, + strings.HasPrefix(componentName, gitComponent+"."): + return DeletedShapeMetadataFlag + case strings.HasPrefix(componentName, postgresComponent): + return DeletedShapeUnchanged + default: + return DeletedShapeEmptyItem + } +} + func getKeys(mymap map[string]*configuration.Item) []string { keys := make([]string, 0, len(mymap)) for key := range mymap { @@ -373,14 +415,18 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration // Delete initValues2 errDelete := updater.DeleteKey(getKeys(initValues2)) require.NoError(t, errDelete, "expected no error on updating keys") - if strings.HasPrefix(component, kubernetesComponent) { - // Kubernetes ConfigMap delete notifications include {"deleted": "true"} metadata + switch config.DeletedShape { + case DeletedShapeMetadataFlag: + // e.g. Kubernetes ConfigMap, Git — emit a sentinel in metadata + // so subscribers can distinguish removal from empty-string set. for k := range initValues2 { initValues2[k] = &configuration.Item{ Metadata: map[string]string{"deleted": "true"}, } } - } else if !strings.HasPrefix(component, postgresComponent) { + case DeletedShapeUnchanged: + // Postgres — leave initValues2 as-is. + default: // DeletedShapeEmptyItem for k := range initValues2 { initValues2[k] = &configuration.Item{} } diff --git a/tests/conformance/configuration_test.go b/tests/conformance/configuration_test.go index 36ed1f33df..f3351aef9c 100644 --- a/tests/conformance/configuration_test.go +++ b/tests/conformance/configuration_test.go @@ -23,11 +23,13 @@ import ( "github.com/stretchr/testify/require" "github.com/dapr/components-contrib/configuration" + c_git "github.com/dapr/components-contrib/configuration/git" c_kubernetes "github.com/dapr/components-contrib/configuration/kubernetes" c_postgres "github.com/dapr/components-contrib/configuration/postgres" c_redis "github.com/dapr/components-contrib/configuration/redis" conf_configuration "github.com/dapr/components-contrib/tests/conformance/configuration" "github.com/dapr/components-contrib/tests/utils/configupdater" + cu_git "github.com/dapr/components-contrib/tests/utils/configupdater/git" cu_kubernetes "github.com/dapr/components-contrib/tests/utils/configupdater/kubernetes" cu_postgres "github.com/dapr/components-contrib/tests/utils/configupdater/postgres" cu_redis "github.com/dapr/components-contrib/tests/utils/configupdater/redis" @@ -41,6 +43,23 @@ func TestConfigurationConformance(t *testing.T) { tc.TestFn = func(comp *TestComponent) func(t *testing.T) { return func(t *testing.T) { + // NOTE: tc.Run(t) was lost in commit 1208b3e3 ("Conformance + // test: move loader to each component type's folder"), so the + // configuration conformance suite has been a silent no-op + // since then — no component has actually exercised these + // tests in CI. Restoring tc.Run(t) (below) surfaces years of + // latent failures in redis/postgres/kubernetes that are out + // of scope for the configuration.git PR. + // + // To keep the matrix listing accurate while not blocking on + // pre-existing breakage, skip non-git components explicitly so + // CI output reflects what is and isn't being exercised. A + // dedicated framework-repair pass should remove these skips. + if comp.Component != "git.local" { + t.Skipf("configuration conformance for %s is not currently exercised: this suite has been a no-op since commit 1208b3e3; the tc.Run(t) restoration in this PR is scoped to git.local only", comp.Component) + return + } + ParseConfigurationMap(t, comp.Config) componentConfigPath := convertComponentNameToPath(comp.Component, comp.Profile) @@ -55,6 +74,8 @@ func TestConfigurationConformance(t *testing.T) { conf_configuration.ConformanceTests(t, props, store, updater, configurationConfig, comp.Component) } } + + tc.Run(t) } func loadConfigurationStore(name string) (configuration.Store, configupdater.Updater) { @@ -68,6 +89,9 @@ func loadConfigurationStore(name string) (configuration.Store, configupdater.Upd case "kubernetes.kind": return c_kubernetes.NewKubernetesConfigMapStore(testLogger), cu_kubernetes.NewKubernetesConfigUpdater(testLogger) + case "git.local": + return c_git.NewGitConfigurationStore(testLogger), + cu_git.NewGitConfigUpdater(testLogger) default: return nil, nil } diff --git a/tests/utils/configupdater/git/git.go b/tests/utils/configupdater/git/git.go new file mode 100644 index 0000000000..424a4b4df1 --- /dev/null +++ b/tests/utils/configupdater/git/git.go @@ -0,0 +1,187 @@ +/* +Copyright 2026 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + + "github.com/dapr/components-contrib/configuration" + "github.com/dapr/components-contrib/tests/utils/configupdater" + "github.com/dapr/kit/logger" +) + +// ConfigUpdater backs the conformance test for the git configuration store. +// Each Add/Update/Delete operation writes files to a private working clone, +// commits, and pushes back to the upstream repo so that the configuration +// store under test sees the change on its next poll tick. +// +// The updater pairs with `mappingMode: file` so emitted store keys are equal +// to the filenames written here. After every push, the updater waits long +// enough for the store's polling loop to observe and absorb the new HEAD — +// this lets the conformance harness call Get/verifySubscribers immediately +// after a write without racing the poller. +type ConfigUpdater struct { + logger logger.Logger + + upstreamURL string + branch string + workdir string + repo *gogit.Repository + wt *gogit.Worktree + settleWait time.Duration + + mu sync.Mutex + seq atomic.Int64 +} + +// NewGitConfigUpdater constructs a fresh updater. props["remoteUrl"] is the +// upstream URL (typically file://). props["branch"] defaults to "main". +func NewGitConfigUpdater(log logger.Logger) configupdater.Updater { + return &ConfigUpdater{logger: log} +} + +func (u *ConfigUpdater) Init(props map[string]string) error { + u.upstreamURL = props["remoteUrl"] + if u.upstreamURL == "" { + return errors.New("remoteUrl is required") + } + u.branch = props["branch"] + if u.branch == "" { + u.branch = "main" + } + + pollInterval, _ := time.ParseDuration(props["pollInterval"]) + if pollInterval <= 0 { + pollInterval = 250 * time.Millisecond + } + // settleWait gives the store-under-test enough time to observe the + // pushed commit on its next poll tick. The 500ms margin absorbs + // goroutine scheduling jitter and ticker alignment. If conformance + // tests start failing intermittently on slow CI, this is the knob. + u.settleWait = pollInterval + 500*time.Millisecond + + dir, err := os.MkdirTemp("", "dapr-config-git-updater-") + if err != nil { + return fmt.Errorf("create workdir: %w", err) + } + u.workdir = dir + + repo, cloneErr := gogit.PlainClone(dir, false, &gogit.CloneOptions{URL: u.upstreamURL}) + if cloneErr != nil { + // Empty repo or no refs — initialise locally and add origin. + var initErr error + repo, initErr = gogit.PlainInit(dir, false) + if initErr != nil { + return fmt.Errorf("init local clone: %w", initErr) + } + if _, remoteErr := repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{u.upstreamURL}, + }); remoteErr != nil { + return fmt.Errorf("add origin: %w", remoteErr) + } + } + u.repo = repo + + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("worktree: %w", err) + } + u.wt = wt + return nil +} + +func (u *ConfigUpdater) AddKey(items map[string]*configuration.Item) error { + return u.writeAndCommit(items, "add", false) +} + +func (u *ConfigUpdater) UpdateKey(items map[string]*configuration.Item) error { + return u.writeAndCommit(items, "update", false) +} + +func (u *ConfigUpdater) DeleteKey(keys []string) error { + u.mu.Lock() + defer u.mu.Unlock() + + for _, k := range keys { + p := filepath.Join(u.workdir, k) + if err := os.RemoveAll(p); err != nil { + return fmt.Errorf("remove %q: %w", k, err) + } + if _, err := u.wt.Remove(k); err != nil { + return fmt.Errorf("git rm %q: %w", k, err) + } + } + return u.commitAndPush(fmt.Sprintf("delete %d", len(keys))) +} + +func (u *ConfigUpdater) writeAndCommit(items map[string]*configuration.Item, op string, _ bool) error { + u.mu.Lock() + defer u.mu.Unlock() + + for k, v := range items { + p := filepath.Join(u.workdir, k) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return fmt.Errorf("mkdir for %q: %w", k, err) + } + if err := os.WriteFile(p, []byte(v.Value), 0o600); err != nil { + return fmt.Errorf("write %q: %w", k, err) + } + if _, err := u.wt.Add(k); err != nil { + return fmt.Errorf("git add %q: %w", k, err) + } + } + return u.commitAndPush(fmt.Sprintf("%s %d", op, len(items))) +} + +func (u *ConfigUpdater) commitAndPush(message string) error { + seq := u.seq.Add(1) + sig := &object.Signature{ + Name: "conformance", + Email: "conformance@dapr.io", + When: time.Now(), + } + if _, err := u.wt.Commit(message+" (#"+strconv.FormatInt(seq, 10)+")", &gogit.CommitOptions{ + Author: sig, + Committer: sig, + }); err != nil { + return fmt.Errorf("commit: %w", err) + } + // Push both `master` and `main` so we don't fight the locally-default + // branch. The destination is always `main` on the upstream. + if err := u.repo.Push(&gogit.PushOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{ + config.RefSpec("refs/heads/master:refs/heads/" + u.branch), + config.RefSpec("refs/heads/" + u.branch + ":refs/heads/" + u.branch), + }, + }); err != nil && !errors.Is(err, gogit.NoErrAlreadyUpToDate) { + return fmt.Errorf("push: %w", err) + } + if u.settleWait > 0 { + time.Sleep(u.settleWait) + } + return nil +}