feat(actions): support referencing ${{ needs... }} variables in runs-on (#10308)

Allows referencing the outputs of previously executed jobs in the `runs-on` field directly by a `${{ needs.some-job.outputs.some-output }}`, and also *indirectly* through the job's `strategy.matrix`.  At its most complicated, supports a workflow with dynamic matrices like this:

```yaml
jobs:
  define-matrix:
    runs-on: docker
    outputs:
      array-value: ${{ steps.define.outputs.array }}
    steps:
      - id: define
        run: |
          echo 'array=["debian-bookworm", "debian-trixie"]' >> "$FORGEJO_OUTPUT"

  runs-on-dynamic-matrix:
    needs: define-matrix
    strategy:
      matrix:
        my-runners: ${{ fromJSON(needs.define-matrix.outputs.array-value) }}
    runs-on: ${{ matrix.my-runners }}
    steps:
      - run: uname -a
```

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.
    - Documentation already (incorrectly) states that `jobs.<job-id>.runs-on` can access the `needs` context. 😛  https://forgejo.org/docs/latest/user/actions/reference/#availability

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

<!--start release-notes-assistant-->

## Release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/10308): <!--number 10308 --><!--line 0 --><!--description ZmVhdChhY3Rpb25zKTogc3VwcG9ydCByZWZlcmVuY2luZyBgJHt7IG5lZWRzLi4uIH19YCB2YXJpYWJsZXMgaW4gYHJ1bnMtb25g-->feat(actions): support referencing `${{ needs... }}` variables in `runs-on`<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10308
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>

models/actions/pre_execution_errors.go

@@ -18,9 +18,14 @@ type PreExecutionError int64
 const (
 	ErrorCodeEventDetectionError PreExecutionError = iota + 1
 	ErrorCodeJobParsingError
-	ErrorCodePersistentIncompleteMatrix
+	ErrorCodePersistentIncompleteMatrix // obsolete
 	ErrorCodeIncompleteMatrixMissingJob
 	ErrorCodeIncompleteMatrixMissingOutput
+	ErrorCodeIncompleteMatrixUnknownCause
+	ErrorCodeIncompleteRunsOnMissingJob
+	ErrorCodeIncompleteRunsOnMissingOutput
+	ErrorCodeIncompleteRunsOnMissingMatrixDimension
+	ErrorCodeIncompleteRunsOnUnknownCause
 )
 
 func TranslatePreExecutionError(lang translation.Locale, run *ActionRun) string {
@@ -42,6 +47,16 @@ func TranslatePreExecutionError(lang translation.Locale, run *ActionRun) string
 		return lang.TrString("actions.workflow.incomplete_matrix_missing_job", run.PreExecutionErrorDetails...)
 	case ErrorCodeIncompleteMatrixMissingOutput:
 		return lang.TrString("actions.workflow.incomplete_matrix_missing_output", run.PreExecutionErrorDetails...)
+	case ErrorCodeIncompleteMatrixUnknownCause:
+		return lang.TrString("actions.workflow.incomplete_matrix_unknown_cause", run.PreExecutionErrorDetails...)
+	case ErrorCodeIncompleteRunsOnMissingJob:
+		return lang.TrString("actions.workflow.incomplete_runson_missing_job", run.PreExecutionErrorDetails...)
+	case ErrorCodeIncompleteRunsOnMissingOutput:
+		return lang.TrString("actions.workflow.incomplete_runson_missing_output", run.PreExecutionErrorDetails...)
+	case ErrorCodeIncompleteRunsOnMissingMatrixDimension:
+		return lang.TrString("actions.workflow.incomplete_runson_missing_matrix_dimension", run.PreExecutionErrorDetails...)
+	case ErrorCodeIncompleteRunsOnUnknownCause:
+		return lang.TrString("actions.workflow.incomplete_runson_unknown_cause", run.PreExecutionErrorDetails...)
 	}
 	return fmt.Sprintf("<unsupported error: code=%v details=%#v", run.PreExecutionErrorCode, run.PreExecutionErrorDetails)
 }

models/actions/pre_execution_errors_test.go

@@ -70,6 +70,46 @@ func TestTranslatePreExecutionError(t *testing.T) {
 			},
 			expected: "Unable to evaluate `strategy.matrix` of job blocked_job: job other_job is not in the `needs` list of job blocked_job (needs-1, needs-2).",
 		},
+		{
+			name: "ErrorCodeIncompleteMatrixUnknownCause",
+			run: &ActionRun{
+				PreExecutionErrorCode:    ErrorCodeIncompleteMatrixUnknownCause,
+				PreExecutionErrorDetails: []any{"blocked_job"},
+			},
+			expected: "Unable to evaluate `strategy.matrix` of job blocked_job: unknown error.",
+		},
+		{
+			name: "ErrorCodeIncompleteRunsOnMissingOutput",
+			run: &ActionRun{
+				PreExecutionErrorCode:    ErrorCodeIncompleteRunsOnMissingOutput,
+				PreExecutionErrorDetails: []any{"blocked_job", "other_job", "some_output"},
+			},
+			expected: "Unable to evaluate `runs-on` of job blocked_job: job other_job does not have an output some_output.",
+		},
+		{
+			name: "ErrorCodeIncompleteRunsOnMissingJob",
+			run: &ActionRun{
+				PreExecutionErrorCode:    ErrorCodeIncompleteRunsOnMissingJob,
+				PreExecutionErrorDetails: []any{"blocked_job", "other_job", "needs-1, needs-2"},
+			},
+			expected: "Unable to evaluate `runs-on` of job blocked_job: job other_job is not in the `needs` list of job blocked_job (needs-1, needs-2).",
+		},
+		{
+			name: "ErrorCodeIncompleteRunsOnMissingMatrixDimension",
+			run: &ActionRun{
+				PreExecutionErrorCode:    ErrorCodeIncompleteRunsOnMissingMatrixDimension,
+				PreExecutionErrorDetails: []any{"blocked_job", "platfurm"},
+			},
+			expected: "Unable to evaluate `runs-on` of job blocked_job: matrix dimension platfurm does not exist.",
+		},
+		{
+			name: "ErrorCodeIncompleteRunsOnUnknownCause",
+			run: &ActionRun{
+				PreExecutionErrorCode:    ErrorCodeIncompleteRunsOnUnknownCause,
+				PreExecutionErrorDetails: []any{"blocked_job"},
+			},
+			expected: "Unable to evaluate `runs-on` of job blocked_job: unknown error.",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

models/actions/run.go

@@ -341,7 +341,7 @@ func InsertRunJobs(ctx context.Context, run *ActionRun, jobs []*jobparser.Single
 			}
 			payload, _ = v.Marshal()
 
-			if len(needs) > 0 || run.NeedApproval || v.IncompleteMatrix {
+			if len(needs) > 0 || run.NeedApproval || v.IncompleteMatrix || v.IncompleteRunsOn {
 				status = StatusBlocked
 			} else {
 				status = StatusWaiting

models/actions/run_job.go

@@ -289,3 +289,13 @@ func (job *ActionRunJob) IsIncompleteMatrix() (bool, *jobparser.IncompleteNeeds,
 	}
 	return jobWorkflow.IncompleteMatrix, jobWorkflow.IncompleteMatrixNeeds, nil
 }
+
+// Checks whether the target job has a `runs-on` field with an expression that requires an input from another job.  The
+// job will be blocked until the other job is complete, and then regenerated and deleted.
+func (job *ActionRunJob) IsIncompleteRunsOn() (bool, *jobparser.IncompleteNeeds, *jobparser.IncompleteMatrix, error) {
+	jobWorkflow, err := job.decodeWorkflowPayload()
+	if err != nil {
+		return false, nil, nil, fmt.Errorf("failure decoding workflow payload: %w", err)
+	}
+	return jobWorkflow.IncompleteRunsOn, jobWorkflow.IncompleteRunsOnNeeds, jobWorkflow.IncompleteRunsOnMatrix, nil
+}

models/actions/run_job_test.go

@@ -189,3 +189,51 @@ func TestActionRunJob_IsIncompleteMatrix(t *testing.T) {
 		})
 	}
 }
+
+func TestActionRunJob_IsIncompleteRunsOn(t *testing.T) {
+	tests := []struct {
+		name         string
+		job          ActionRunJob
+		isIncomplete bool
+		needs        *jobparser.IncompleteNeeds
+		matrix       *jobparser.IncompleteMatrix
+		errContains  string
+	}{
+		{
+			name:         "normal workflow",
+			job:          ActionRunJob{WorkflowPayload: []byte("name: workflow")},
+			isIncomplete: false,
+		},
+		{
+			name:         "nincomplete_runs_on workflow",
+			job:          ActionRunJob{WorkflowPayload: []byte("name: workflow\nincomplete_runs_on: true\nincomplete_runs_on_needs: { job: abc }")},
+			needs:        &jobparser.IncompleteNeeds{Job: "abc"},
+			isIncomplete: true,
+		},
+		{
+			name:         "nincomplete_runs_on workflow",
+			job:          ActionRunJob{WorkflowPayload: []byte("name: workflow\nincomplete_runs_on: true\nincomplete_runs_on_matrix: { dimension: abc }")},
+			matrix:       &jobparser.IncompleteMatrix{Dimension: "abc"},
+			isIncomplete: true,
+		},
+		{
+			name:        "unparseable workflow",
+			job:         ActionRunJob{WorkflowPayload: []byte("name: []\nincomplete_runs_on: true")},
+			errContains: "failure unmarshaling WorkflowPayload to SingleWorkflow: yaml: unmarshal errors",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			isIncomplete, needs, matrix, err := tt.job.IsIncompleteRunsOn()
+			if tt.errContains != "" {
+				assert.ErrorContains(t, err, tt.errContains)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.isIncomplete, isIncomplete)
+				assert.Equal(t, tt.needs, needs)
+				assert.Equal(t, tt.matrix, matrix)
+			}
+		})
+	}
+}

models/actions/run_test.go

@@ -238,3 +238,37 @@ jobs:
 	// Expect job with an incomplete matrix to be StatusBlocked:
 	assert.Equal(t, StatusBlocked, job.Status)
 }
+
+func TestActionRun_IncompleteRunsOn(t *testing.T) {
+	require.NoError(t, unittest.PrepareTestDatabase())
+
+	pullRequestPosterID := int64(4)
+	repoID := int64(10)
+	pullRequestID := int64(2)
+	runDoesNotNeedApproval := &ActionRun{
+		RepoID:              repoID,
+		PullRequestID:       pullRequestID,
+		PullRequestPosterID: pullRequestPosterID,
+	}
+
+	workflowRaw := []byte(`
+jobs:
+  job2:
+    runs-on: ${{ needs.other-job.outputs.some-output }}
+    steps:
+      - run: true
+`)
+	workflows, err := jobparser.Parse(workflowRaw, false, jobparser.WithJobOutputs(map[string]map[string]string{}), jobparser.SupportIncompleteRunsOn())
+	require.NoError(t, err)
+	require.True(t, workflows[0].IncompleteRunsOn) // must be set for this test scenario to be valid
+
+	require.NoError(t, InsertRun(t.Context(), runDoesNotNeedApproval, workflows))
+
+	jobs, err := db.Find[ActionRunJob](t.Context(), FindRunJobOptions{RunID: runDoesNotNeedApproval.ID})
+	require.NoError(t, err)
+	require.Len(t, jobs, 1)
+	job := jobs[0]
+
+	// Expect job with an incomplete runs-on to be StatusBlocked:
+	assert.Equal(t, StatusBlocked, job.Status)
+}