fix: terminate git (& other) subcommands gracefully on context timeout/cancellation (#13263)

By default, when you create an `exec.CommandContext` in Go, when the context is cancelled or timed out, Go will SIGKILL the subprocess which does not allow the subprocess to perform any cleanup work.  As some git operations in Forgejo will be linked to the current HTTP context, and the current HTTP context will be closed when the HTTP client disconnects, this can theoretically cause SIGKILLs when locks are being held.

This PR contains two fixes:
- Change the behaviour of all subprocesses that Forgejo creates to use a graceful termination, SIGTERM'ing the process, and then SIGKILL'ing it after a timeout.
- Fixes a bug where previously all subprocesses are setup in a process group (by 1d04e8641d), but they were never *killed* in a process group, preventing child processes from receiving these termination signals.  Both SIGTERM and SIGKILL are sent to the process group.

Introduces new configuration setting which controls subcommand graceful timeouts.  If a subprocess does not shutdown from SIGTERM within `[server].SUBPROCESS_TERMINATE_GRACE` (default 5 seconds), then an error will be logged, and the process will be SIGKILL'd.  The error is:
```
2026/07/01 08:31:20 modules/log/init.go:38:func2() [E] Subprocess pid=3790963 (/nix/store/c0277k5giric1mn9dklllavbzvxl6hzb-git-2.53.0/bin/git blame --porcelain 8ee7e7c32b -- options/locale/locale_en-US.ini) failed to terminate from SIGTERM after grace period 0s, and was sent SIGKILL. The subprocess termination may leave incomplete state on the server, such as lock files. `[server].SUBPROCESS_TERMINATE_GRACE` can be changed to give subprocesses more time to cleanly exit.
```

(This was produced on a live server by setting SUBPROCESS_TERMINATE_GRACE to zero seconds, loading a page that does a `git blame`, and hitting escape in the browser quickly.)

### Tests for Go changes

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I ran...
  - [x] `make pr-go` before pushing

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
    - Will create a documentation PR with new config settings.
- [ ] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change.
- [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/13263
Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org>

modules/process/manager_exec.go

@@ -26,18 +26,17 @@ func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc
 	return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...)
 }
 
-// ExecDirEnv runs a command in given path and environment variables, and waits for its completion
-// up to the given timeout (or DefaultTimeout if -1 is given).
-// Returns its complete stdout and stderr
-// outputs and an error, if any (including timeout)
+// ExecDirEnv runs a command in given path and environment variables, and waits for its completion up to the given
+// timeout (or 60s if -1 is given). Returns its complete stdout and stderr outputs and an error, if any (including
+// timeout)
 func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
 	return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...)
 }
 
-// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion
-// up to the given timeout (or DefaultTimeout if timeout <= 0 is given).
-// Returns its complete stdout and stderr
-// outputs and an error, if any (including timeout)
+// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its
+// completion up to the given timeout (or 60s if timeout <= 0 is given).  If the process times out, will send SIGTERM
+// and wait 5s before the process is SIGKILL'd.  Returns its complete stdout and stderr outputs and an error, if any
+// (including timeout)
 func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) {
 	if timeout <= 0 {
 		timeout = 60 * time.Second
@@ -57,7 +56,7 @@ func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, d
 	if stdIn != nil {
 		cmd.Stdin = stdIn
 	}
-	SetSysProcAttribute(cmd)
+	SetupCancellableCommand(cmd)
 
 	if err := cmd.Start(); err != nil {
 		return "", "", err

modules/process/manager_test.go

@@ -5,10 +5,15 @@ package process
 
 import (
 	"context"
+	"os"
+	"os/exec"
+	"path"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetManager(t *testing.T) {
@@ -109,3 +114,154 @@ func TestExecTimeoutAlways(t *testing.T) {
 		}
 	}
 }
+
+func TestSetupCancellableCommand(t *testing.T) {
+	t.Run("context cancellation gives process time to clean itself up", func(t *testing.T) {
+		script := `
+		trap 'rm -f lock; exit 0' TERM
+		touch lock
+		touch created-lock
+		while true;
+		do
+			sleep 0.05
+		done
+		`
+		timeout := 15 * time.Second
+		tmpdir := t.TempDir()
+
+		childCtx, cancel := context.WithCancel(t.Context())
+
+		wg := sync.WaitGroup{}
+		wg.Go(func() {
+			_, _, err := GetManager().ExecDirEnvStdIn(childCtx, timeout, tmpdir, t.Name(), nil, nil, "bash", "-c", script)
+			require.ErrorContains(t, err, "context canceled")
+		})
+
+		// 'created-lock' file should exist, indicating that the test script had created the lock file
+		require.Eventually(t, func() bool {
+			_, err := os.Stat(path.Join(tmpdir, "created-lock"))
+			return err == nil
+		}, time.Second, time.Millisecond)
+
+		cancel()  // terminate the script by killing childCtx
+		wg.Wait() // wait for Go to recognize process is killed
+
+		// 'lock' file should not exist, as it should be cleaned up by the TERM trap
+		_, err := os.Stat(path.Join(tmpdir, "lock"))
+		require.Error(t, err, "lock file must not exist")
+		assert.True(t, os.IsNotExist(err), "lock file must not exist")
+	})
+
+	t.Run("child processes are killed", func(t *testing.T) {
+		// It's a bit tricky to observe the state of a grandchild process, especially when we're going to turn it into a
+		// zombie.  os.FindProcess+p.Kill(0) are the documented way of checking whether a process is running, which we
+		// could do based upon child-pid... but the process will be a zombie once its parent is killed without waiting
+		// for it, and zombie processes absorb that kill with no response.
+		//
+		// The strategy used here is a tweak of forgejo-runner's `TestCancelLongRunningCommand` test, which faced the
+		// same problem.  The subprocess writes a heartbeat file.  We delete it when we expect the process is killed,
+		// and then verify it never reappears.
+		script := `
+		sh -c 'echo $$ > child-pid; while true; do touch child-heartbeat; sleep 0.05; done' &
+		touch ready
+		wait
+		`
+		timeout := 15 * time.Second
+		tmpdir := t.TempDir()
+
+		childCtx, cancel := context.WithCancel(t.Context())
+
+		wg := sync.WaitGroup{}
+		wg.Go(func() {
+			_, _, err := GetManager().ExecDirEnvStdIn(childCtx, timeout, tmpdir, t.Name(), nil, nil, "bash", "-c", script)
+			require.ErrorContains(t, err, "context canceled")
+		})
+
+		// Wait for 'child-heartbeat' to exist indicating that the subprocess is running
+		// childPidFile := path.Join(tmpdir, "child-pid")
+		childHeartbeat := path.Join(tmpdir, "child-heartbeat")
+		require.Eventually(t, func() bool {
+			_, err := os.Stat(childHeartbeat)
+			return err == nil
+		}, time.Second, time.Millisecond)
+
+		cancel()  // terminate the script by killing childCtx
+		wg.Wait() // wait for Go to recognize process is killed
+
+		require.NoError(t, os.Remove(childHeartbeat)) // remove heartbeat file
+
+		// wait 2x the sleep period in the child process (50ms -> 100ms)
+		time.Sleep(100 * time.Millisecond)
+
+		// Verify that child heartbeat is not recreated, confirming the process was killed
+		_, err := os.Stat(childHeartbeat)
+		require.Error(t, err, "heartbeat file must not exist")
+		assert.True(t, os.IsNotExist(err), "heartbeat file must not exist")
+	})
+
+	t.Run("fallback to SIGKILL", func(t *testing.T) {
+		// cannot use test.MockVariableValue because it causes an import cycle, `process` is too low-level of a module;
+		// t.Cleanup is used to restore the global values instead.
+		originalTerminateGraceTimeout := TerminateGraceTimeout
+		TerminateGraceTimeout = 100 * time.Millisecond
+
+		originalNotifyTerminateGraceExhausted := NotifyTerminateGraceExhausted
+		notifyOutput := []*exec.Cmd{}
+		NotifyTerminateGraceExhausted = func(cmd *exec.Cmd) {
+			notifyOutput = append(notifyOutput, cmd)
+		}
+
+		t.Cleanup(func() {
+			TerminateGraceTimeout = originalTerminateGraceTimeout
+			NotifyTerminateGraceExhausted = originalNotifyTerminateGraceExhausted
+		})
+
+		script := `
+		trap 'touch term-step-1; sleep 30; touch term-step-2 exit 0' TERM
+		touch script-running
+		while true;
+		do
+			sleep 0.05
+		done
+		`
+
+		timeout := 15 * time.Second
+		tmpdir := t.TempDir()
+		childCtx, cancel := context.WithCancel(t.Context())
+
+		wg := sync.WaitGroup{}
+		wg.Go(func() {
+			_, _, err := GetManager().ExecDirEnvStdIn(childCtx, timeout, tmpdir, t.Name(), nil, nil, "bash", "-c", script)
+			require.ErrorContains(t, err, "context canceled")
+		})
+
+		// 'script-running' file should exist indicating that the trap on SIGTERM is set
+		require.Eventually(t, func() bool {
+			_, err := os.Stat(path.Join(tmpdir, "script-running"))
+			return err == nil
+		}, time.Second, time.Millisecond)
+
+		beforeCancel := time.Now()
+		assert.Empty(t, notifyOutput)
+		cancel()  // terminate the script by killing childCtx
+		wg.Wait() // wait for Go to recognize process is killed
+
+		// Should take at least TerminateGraceTimeout to perform the cancel
+		cancelDuration := time.Since(beforeCancel)
+		assert.GreaterOrEqual(t, cancelDuration, TerminateGraceTimeout)
+
+		// 'term-step-1' file should exist, indicating that SIGTERM was received
+		_, err := os.Stat(path.Join(tmpdir, "term-step-1"))
+		require.NoError(t, err)
+
+		// 'term-step-2' file must not exist, as that would indicate that the script ran for the full "sleep 30" and wasn't SIGKILL'd
+		_, err = os.Stat(path.Join(tmpdir, "term-step-2"))
+		require.Error(t, err, "term-step-2 file must not exist")
+		assert.True(t, os.IsNotExist(err), "term-step-2 file must not exist")
+
+		// notifyOutput should contain the command that was run
+		require.Len(t, notifyOutput, 1)
+		cmdRun := notifyOutput[0]
+		assert.Contains(t, cmdRun.Path, "bash")
+	})
+}

modules/process/manager_unix.go

@@ -1,15 +1,118 @@
-// Copyright 2022 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
+// Copyright 2026 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: GPLv3-or-later
 
 package process
 
 import (
+	"os"
 	"os/exec"
 	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
 )
 
-// SetSysProcAttribute sets the common SysProcAttrs for commands
-func SetSysProcAttribute(cmd *exec.Cmd) {
-	// When Gitea runs SubProcessA -> SubProcessB and SubProcessA gets killed by context timeout, use setpgid to make sure the sub processes can be reaped instead of leaving defunct(zombie) processes.
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+// A command run through [SetupCancellableCommand] will be sent SIGKILL after this grace period if the processes do not
+// terminate after SIGTERM is sent to them.
+var TerminateGraceTimeout = 5 * time.Second
+
+// This closure is invoked in the event that SIGKILL is sent after a grace period has expired.  Intended for logging the
+// situation, which cannot be performed inside the process module as it would cause a cyclical dependency on Forgejo's
+// log module.
+var NotifyTerminateGraceExhausted func(cmd *exec.Cmd)
+
+// Identify errors that can occur in kill or wait that indicate the kill was actually successful as the process doesn't
+// exist anymore.
+func killErrorSafeToIgnore(err error) bool {
+	errno, isErrno := err.(syscall.Errno)
+	if isErrno {
+		switch errno {
+		case syscall.ESRCH, // No such process
+			syscall.ECHILD: // No child processes
+			return true
+		}
+	}
+	syscallErr, isSyscallErr := err.(*os.SyscallError) // errno may be wrapped in SyscallError
+	if isSyscallErr && killErrorSafeToIgnore(syscallErr.Err) {
+		return true
+	}
+	return false
+}
+
+// Configures an [exec.Cmd] so that its Cancel function will gracefully shutdown the process and all subprocesses by
+// sending them SIGTERM.  If they do not shutdown from the SIGTERM in the [TerminateGraceTimeout] grace period, they
+// will be SIGKILL'd instead. Note that `SysProcAttr` and `Cancel` on the provided command are overwritten, and the
+// command's `WaitDelay` will no longer have an impact.
+func SetupCancellableCommand(cmd *exec.Cmd) {
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		// When running SubProcessA -> SubProcessB and SubProcessA gets killed by context timeout, use setpgid to make
+		// sure the sub processes can be reaped instead of leaving defunct(zombie) processes.
+		Setpgid: true,
+	}
+
+	// Override the default behaviour, which would have been to SIGKILL the process and wouldn't allow subprocesses the
+	// ability to gracefully shutdown.  This could have lead to git subprocesses leaving lock files not cleaned up.
+	cmd.Cancel = func() error {
+		// We're going to TERM/KILL this process, but we cannot wait() on the PID within Cancel -- exactly one goroutine
+		// may wait() the child process, and if we do it in here we'll cause random errors in the other goroutine.  A
+		// PID FD is used to be able to get woken when the process dies, instead.
+		//
+		// It would be nice to use SysProcAttr to get the pidfd automatically, which would remove this syscall and
+		// there'd be no chance of an error here.  But we wouldn't be able to close that FD if the process wasn't
+		// cancelled since SetupCancellableCommand doesn't interact with commands that don't cancel.
+		pidfd, err := unix.PidfdOpen(cmd.Process.Pid, 0)
+		if killErrorSafeToIgnore(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		defer unix.Close(pidfd)
+
+		// When cmd is being killed (typically by Go when CommandContext's ctx is cancelled), send SIGTERM rather than
+		// the default of SIGKILL.  Send it to the entire process group by sending to the negative PID.
+		err = syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM)
+		if killErrorSafeToIgnore(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		// Begin waiting for the process to finish by polling the pidfd.  This is a bit ugly to have to implement here,
+		// but as noted earlier we can't just `cmd.Process.Wait()` because another goroutine will be doing that
+		// concurrently.  We need to repeatedly call Poll() until it indicates that the pidfd is readable, recalculating
+		// remaining time in the grace period -- the repeated calls are needed because EINTR can interrupt the poll
+		// briefly.
+		pfds := []unix.PollFd{{Fd: int32(pidfd), Events: unix.POLLIN}}
+		deadline := time.Now().Add(TerminateGraceTimeout)
+		for {
+			timeout := int(time.Until(deadline).Milliseconds())
+			if timeout < 0 {
+				// Grace period expired
+				break
+			}
+			n, err := unix.Poll(pfds, timeout)
+			if err == unix.EINTR {
+				continue // interrupted by a signal; recompute remaining time and retry
+			}
+			if err != nil {
+				return err
+			}
+			if n > 0 {
+				// pidfd is readable, indicating that the process is terminated.  We don't bother actually reading the
+				// exit code from the fd -- the other goroutine that is Wait()'ing will do that.
+				return nil
+			}
+		}
+
+		// After sending SIGTERM, we hit the terminate grace timeout and the subprocess is still running.  Send SIGKILL
+		// to the process group as a fallback.
+		if NotifyTerminateGraceExhausted != nil {
+			NotifyTerminateGraceExhausted(cmd)
+		}
+		err = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+		if killErrorSafeToIgnore(err) {
+			return nil
+		}
+		return err
+	}
 }