fix: resolve Git SSH signature forwarding failures (#645)

cmd/agent/container/credentials_server.go

@@ -135,21 +135,26 @@ func (cmd *CredentialsServerCmd) Run(ctx context.Context, port int) error {
 		}(cmd.User)
 	}
 
-	// configure git ssh signature helper
+	// configure git ssh signature helper — non-fatal so that a signing
+	// setup failure does not take down the entire credentials server
+	// (git/docker credential forwarding, port forwarding, etc.)
 	if cmd.GitUserSigningKey != "" {
 		decodedKey, err := base64.StdEncoding.DecodeString(cmd.GitUserSigningKey)
 		if err != nil {
-			return fmt.Errorf("decode git ssh signature key: %w", err)
-		}
-		err = gitsshsigning.ConfigureHelper(cmd.User, string(decodedKey), log)
-		if err != nil {
-			return fmt.Errorf("configure git ssh signature helper: %w", err)
+			log.Errorf("Failed to decode git SSH signing key, signing will be unavailable: %v", err)
+		} else {
+			err = gitsshsigning.ConfigureHelper(cmd.User, string(decodedKey), log)
+			if err != nil {
+				log.Errorf(
+					"Failed to configure git SSH signature helper, signing will be unavailable: %v",
+					err,
+				)
+			} else {
+				defer func(userName string) {
+					_ = gitsshsigning.RemoveHelper(userName)
+				}(cmd.User)
+			}
 		}
-
-		// cleanup when we are done
-		defer func(userName string) {
-			_ = gitsshsigning.RemoveHelper(userName)
-		}(cmd.User)
 	}
 
 	return credentials.RunCredentialsServer(ctx, port, tunnelClient, log)

cmd/agent/git_ssh_signature_test.go

@@ -55,3 +55,25 @@ func (s *GitSSHSignatureTestSuite) TestParseDefaultsToSign() {
 	assert.Equal(s.T(), "sign", result.command)
 	assert.Equal(s.T(), "/tmp/buffer", result.bufferFile)
 }
+
+func (s *GitSSHSignatureTestSuite) TestParseEmptyArgs() {
+	result := parseSSHKeygenArgs([]string{})
+	assert.Equal(s.T(), "sign", result.command)
+	assert.Equal(s.T(), "", result.bufferFile)
+	assert.Equal(s.T(), "", result.certPath)
+	assert.Equal(s.T(), "", result.namespace)
+}
+
+func (s *GitSSHSignatureTestSuite) TestParseMultipleUnknownFlags() {
+	// ssh-keygen may pass several unknown boolean flags; buffer file must still be found.
+	args := []string{"-Y", "sign", "-n", "git", "-f", "/key.pub", "-U", "-O", "/tmp/buf"}
+	result := parseSSHKeygenArgs(args)
+	assert.Equal(s.T(), "/key.pub", result.certPath)
+	assert.Equal(s.T(), "/tmp/buf", result.bufferFile)
+}
+
+func (s *GitSSHSignatureTestSuite) TestParseBufferFileWithSpaces() {
+	args := []string{"-Y", "sign", "-n", "git", "-f", "/key.pub", "/tmp/my buffer file"}
+	result := parseSSHKeygenArgs(args)
+	assert.Equal(s.T(), "/tmp/my buffer file", result.bufferFile)
+}

cmd/ssh.go

@@ -52,6 +52,7 @@ type SSHCmd struct {
 	AgentForwarding           bool
 	GPGAgentForwarding        bool
 	GitSSHSignatureForwarding bool
+	GitSSHSigningKey          string
 
 	// ssh keepalive options
 	SSHKeepAliveInterval time.Duration `json:"sshKeepAliveInterval,omitempty"`
@@ -147,6 +148,9 @@ func NewSSHCmd(f *flags.GlobalFlags) *cobra.Command {
 	sshCmd.Flags().
 		DurationVar(&cmd.SSHKeepAliveInterval, "ssh-keepalive-interval", 55*time.Second,
 			"How often should keepalive request be made (55s)")
+	sshCmd.Flags().
+		StringVar(&cmd.GitSSHSigningKey, "git-ssh-signing-key", "",
+			"The SSH signing key to use for git commit signing inside the workspace")
 	sshCmd.Flags().StringVar(
 		&cmd.TermMode,
 		"term-mode",
@@ -517,6 +521,7 @@ func (cmd *SSHCmd) startTunnel(
 			configureDockerCredentials,
 			configureGitCredentials,
 			configureGitSSHSignatureHelper,
+			cmd.GitSSHSigningKey,
 			log,
 		)
 	}
@@ -652,6 +657,7 @@ func (cmd *SSHCmd) startServices(
 	containerClient *ssh.Client,
 	workspace *provider.Workspace,
 	configureDockerCredentials, configureGitCredentials, configureGitSSHSignatureHelper bool,
+	gitSSHSigningKey string,
 	log log.Logger,
 ) {
 	if cmd.User != "" {
@@ -668,6 +674,7 @@ func (cmd *SSHCmd) startServices(
 				ConfigureDockerCredentials:     configureDockerCredentials,
 				ConfigureGitCredentials:        configureGitCredentials,
 				ConfigureGitSSHSignatureHelper: configureGitSSHSignatureHelper,
+				GitSSHSigningKey:               gitSSHSigningKey,
 				Log:                            log,
 			},
 		)

cmd/up.go

@@ -15,7 +15,6 @@ import (
 	"strings"
 	"syscall"
 
-	"al.essio.dev/pkg/shellescape"
 	"github.com/blang/semver/v4"
 	"github.com/sirupsen/logrus"
 	"github.com/skevetter/devpod/cmd/flags"
@@ -395,17 +394,6 @@ func (cmd *UpCmd) configureWorkspace(
 		return err
 	}
 
-	// Run after dotfiles so the signing config isn't overwritten by a
-	// dotfiles installer that replaces .gitconfig.
-	gitSSHSignatureEnabled := devPodConfig.ContextOption(
-		config.ContextOptionGitSSHSignatureForwarding,
-	) == "true"
-	if cmd.GitSSHSigningKey != "" && gitSSHSignatureEnabled {
-		if err := setupGitSSHSignature(cmd.GitSSHSigningKey, client); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -481,6 +469,7 @@ func (o *ideOpener) open(
 			user,
 			ideOptions,
 			o.cmd.SSHAuthSockID,
+			o.cmd.GitSSHSigningKey,
 			o.log,
 		)
 
@@ -499,6 +488,7 @@ func (o *ideOpener) open(
 			user,
 			ideOptions,
 			o.cmd.SSHAuthSockID,
+			o.cmd.GitSSHSigningKey,
 			o.log,
 		)
 
@@ -511,6 +501,7 @@ func (o *ideOpener) open(
 			user,
 			ideOptions,
 			o.cmd.SSHAuthSockID,
+			o.cmd.GitSSHSigningKey,
 			o.log,
 		)
 
@@ -854,6 +845,7 @@ func startJupyterNotebookInBrowser(
 	user string,
 	ideOptions map[string]config.OptionValue,
 	authSockID string,
+	gitSSHSigningKey string,
 	logger log.Logger,
 ) error {
 	if forwardGpg {
@@ -900,6 +892,7 @@ func startJupyterNotebookInBrowser(
 		false,
 		extraPorts,
 		authSockID,
+		gitSSHSigningKey,
 		logger,
 	)
 }
@@ -912,6 +905,7 @@ func startRStudioInBrowser(
 	user string,
 	ideOptions map[string]config.OptionValue,
 	authSockID string,
+	gitSSHSigningKey string,
 	logger log.Logger,
 ) error {
 	if forwardGpg {
@@ -957,6 +951,7 @@ func startRStudioInBrowser(
 		false,
 		extraPorts,
 		authSockID,
+		gitSSHSigningKey,
 		logger,
 	)
 }
@@ -1005,6 +1000,7 @@ func startVSCodeInBrowser(
 	workspaceFolder, user string,
 	ideOptions map[string]config.OptionValue,
 	authSockID string,
+	gitSSHSigningKey string,
 	logger log.Logger,
 ) error {
 	if forwardGpg {
@@ -1054,6 +1050,7 @@ func startVSCodeInBrowser(
 		forwardPorts,
 		extraPorts,
 		authSockID,
+		gitSSHSigningKey,
 		logger,
 	)
 }
@@ -1150,6 +1147,7 @@ func startBrowserTunnel(
 	forwardPorts bool,
 	extraPorts []string,
 	authSockID string,
+	gitSSHSigningKey string,
 	logger log.Logger,
 ) error {
 	// Setup a backhaul SSH connection using the remote user so there is an AUTH SOCK to use
@@ -1245,6 +1243,7 @@ func startBrowserTunnel(
 					ConfigureDockerCredentials:     configureDockerCredentials,
 					ConfigureGitCredentials:        configureGitCredentials,
 					ConfigureGitSSHSignatureHelper: configureGitSSHSignatureHelper,
+					GitSSHSigningKey:               gitSSHSigningKey,
 					Log:                            logger,
 				},
 			)
@@ -1553,44 +1552,6 @@ func collectDotfilesScriptEnvKeyvaluePairs(envFiles []string) ([]string, error)
 	return keyValues, nil
 }
 
-func setupGitSSHSignature(
-	signingKey string,
-	client client2.BaseWorkspaceClient,
-) error {
-	execPath, err := os.Executable()
-	if err != nil {
-		return err
-	}
-
-	remoteUser, err := devssh.GetUser(
-		client.WorkspaceConfig().ID,
-		client.WorkspaceConfig().SSHConfigPath,
-		client.WorkspaceConfig().SSHConfigIncludePath,
-	)
-	if err != nil {
-		remoteUser = "root"
-	}
-
-	// #nosec G204 -- execPath is from os.Executable(), not user input
-	out, err := exec.Command(
-		execPath,
-		"ssh",
-		"--user",
-		remoteUser,
-		"--context",
-		client.Context(),
-		client.Workspace(),
-		"--command",
-		shellescape.QuoteCommand(
-			[]string{config.BinaryName, "agent", "git-ssh-signature-helper", signingKey},
-		),
-	).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("setup git ssh signature helper: %w, output: %s", err, string(out))
-	}
-	return nil
-}
-
 func performGpgForwarding(
 	client client2.BaseWorkspaceClient,
 	log log.Logger,

e2e/tests/ssh/ssh.go

@@ -144,29 +144,15 @@ var _ = ginkgo.Describe("devpod ssh test suite", ginkgo.Label("ssh"), ginkgo.Ord
 			err = f.DevPodUp(ctx, tempDir, "--git-ssh-signing-key", keyPath+".pub")
 			framework.ExpectNoError(err)
 
-			// Step 1: Verify the helper script was installed and executable
-			out, err := f.DevPodSSH(ctx, tempDir,
-				"test -x /usr/local/bin/devpod-ssh-signature && echo EXISTS",
-			)
-			framework.ExpectNoError(err)
-			gomega.Expect(strings.TrimSpace(out)).To(
-				gomega.Equal("EXISTS"),
-				"devpod-ssh-signature helper script should be installed and executable",
-			)
-
-			// Step 2: Verify git config was written correctly
-			out, err = f.DevPodSSH(ctx, tempDir, "git config --global gpg.ssh.program")
-			framework.ExpectNoError(err)
-			gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("devpod-ssh-signature"))
-
-			out, err = f.DevPodSSH(ctx, tempDir, "git config --global gpg.format")
-			framework.ExpectNoError(err)
-			gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("ssh"))
-
-			// Step 3: Attempt a signed commit with the credentials server
-			// tunnel active. The signing request is forwarded over the tunnel
-			// to the host where ssh-keygen performs the actual signing.
+			// Verify helper installation, git config, and a signed commit
+			// in a single SSH session with --start-services so the
+			// credentials server tunnel is active. The helper is installed
+			// asynchronously by the credentials server, so retry briefly.
 			commitCmd := strings.Join([]string{
+				"for i in $(seq 1 30); do test -x /usr/local/bin/devpod-ssh-signature && break; sleep 1; done",
+				"test -x /usr/local/bin/devpod-ssh-signature",
+				"test \"$(git config --global gpg.ssh.program)\" = devpod-ssh-signature",
+				"test \"$(git config --global gpg.format)\" = ssh",
 				"cd /tmp",
 				"git init test-sign-repo",
 				"cd test-sign-repo",
@@ -178,13 +164,19 @@ var _ = ginkgo.Describe("devpod ssh test suite", ginkgo.Label("ssh"), ginkgo.Ord
 				"git commit -m 'signed test commit' 2>&1",
 			}, " && ")
 
-			stdout, stderr, err := f.ExecCommandCapture(ctx, []string{
+			// The signing key must be passed on each SSH invocation so the
+			// credentials server can configure the helper inside the container.
+			sshBase := []string{
 				"ssh",
 				"--agent-forwarding",
 				"--start-services",
+				"--git-ssh-signing-key", keyPath + ".pub",
 				tempDir,
-				"--command", commitCmd,
-			})
+			}
+
+			stdout, stderr, err := f.ExecCommandCapture(ctx,
+				append(sshBase, "--command", commitCmd),
+			)
 			ginkgo.GinkgoWriter.Printf("commit stdout: %s\n", stdout)
 			ginkgo.GinkgoWriter.Printf("commit stderr: %s\n", stderr)
 			framework.ExpectNoError(err)
@@ -194,9 +186,7 @@ var _ = ginkgo.Describe("devpod ssh test suite", ginkgo.Label("ssh"), ginkgo.Ord
 				"git commit should succeed with the signed test commit message",
 			)
 
-			// Step 4: Verify the commit is actually signed with a valid SSH signature.
-			// Read the public key that was used for signing so we can build
-			// an allowed-signers file inside the workspace for verification.
+			// Verify the commit is signed with a valid SSH signature.
 			pubKeyBytes, err := os.ReadFile(
 				keyPath + ".pub",
 			) // #nosec G304 -- test file with controlled path
@@ -205,20 +195,14 @@ var _ = ginkgo.Describe("devpod ssh test suite", ginkgo.Label("ssh"), ginkgo.Ord
 
 			verifyCmd := strings.Join([]string{
 				"cd /tmp/test-sign-repo",
-				// Create allowed signers file mapping the test email to our public key
 				"echo 'test@example.com " + pubKey + "' > /tmp/allowed_signers",
 				"git config gpg.ssh.allowedSignersFile /tmp/allowed_signers",
-				// Verify the commit signature is valid
 				"git verify-commit HEAD 2>&1",
 			}, " && ")
 
-			stdout, stderr, err = f.ExecCommandCapture(ctx, []string{
-				"ssh",
-				"--agent-forwarding",
-				"--start-services",
-				tempDir,
-				"--command", verifyCmd,
-			})
+			stdout, stderr, err = f.ExecCommandCapture(ctx,
+				append(sshBase, "--command", verifyCmd),
+			)
 			ginkgo.GinkgoWriter.Printf("verify stdout: %s\n", stdout)
 			ginkgo.GinkgoWriter.Printf("verify stderr: %s\n", stderr)
 			framework.ExpectNoError(err)
@@ -232,13 +216,9 @@ var _ = ginkgo.Describe("devpod ssh test suite", ginkgo.Label("ssh"), ginkgo.Ord
 
 			// And confirm the signature log shows the correct principal
 			logCmd := "cd /tmp/test-sign-repo && git log --show-signature -1 2>&1"
-			stdout, stderr, err = f.ExecCommandCapture(ctx, []string{
-				"ssh",
-				"--agent-forwarding",
-				"--start-services",
-				tempDir,
-				"--command", logCmd,
-			})
+			stdout, stderr, err = f.ExecCommandCapture(ctx,
+				append(sshBase, "--command", logCmd),
+			)
 			ginkgo.GinkgoWriter.Printf("log stdout: %s\n", stdout)
 			ginkgo.GinkgoWriter.Printf("log stderr: %s\n", stderr)
 			framework.ExpectNoError(err)

e2e/tests/ssh/testdata/ssh-signing/devcontainer.json

@@ -1,4 +1,5 @@
 {
   "name": "SSH Signing Test",
-  "image": "mcr.microsoft.com/devcontainers/go:1"
+  "image": "mcr.microsoft.com/devcontainers/go:1",
+  "forwardPorts": [8300, 7080]
 }