feat: support container variables

Author: bahrmichael

internal/batches/executor/run_steps.go

@@ -165,10 +165,16 @@ func RunSteps(ctx context.Context, opts *RunStepsOpts) (stepResults []execution.
 			continue
 		}
 
+		resolvedContainer, err := renderStepContainer(step.Container, &stepContext)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to resolve image for step %d", i+1)
+		}
+		step.Container = resolvedContainer
+
 		// We need to grab the digest for the exact image we're using.
 		img, err := opts.EnsureImage(ctx, step.Container)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "failed to pull image for step %d: %s", i+1, step.Container)
 		}
 		digest, err := img.Digest(ctx)
 		if err != nil {
@@ -241,6 +247,27 @@ func RunSteps(ctx context.Context, opts *RunStepsOpts) (stepResults []execution.
 	return stepResults, err
 }
 
+func renderStepContainer(container string, stepContext *template.StepContext) (string, error) {
+	if container == "" {
+		return "", nil
+	}
+
+	var out bytes.Buffer
+	if err := template.RenderStepTemplate("step-container", container, &out, stepContext); err != nil {
+		return "", err
+	}
+
+	resolved := out.String()
+	if strings.TrimSpace(resolved) == "" {
+		return "", errors.New("empty image")
+	}
+	if strings.Contains(resolved, "${{") {
+		return "", errors.Errorf("unresolved template in image %q", resolved)
+	}
+
+	return resolved, nil
+}
+
 const workDir = "/work"
 
 func executeSingleStep(

internal/batches/executor/run_steps_test.go

@@ -0,0 +1,30 @@
+package executor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/sourcegraph/sourcegraph/lib/batches/template"
+)
+
+func TestRenderStepContainer(t *testing.T) {
+	t.Run("static image", func(t *testing.T) {
+		got, err := renderStepContainer("alpine:3", &template.StepContext{})
+		require.NoError(t, err)
+		require.Equal(t, "alpine:3", got)
+	})
+
+	t.Run("output image", func(t *testing.T) {
+		got, err := renderStepContainer("${{ outputs.imageName }}", &template.StepContext{
+			Outputs: map[string]any{"imageName": "alpine:3"},
+		})
+		require.NoError(t, err)
+		require.Equal(t, "alpine:3", got)
+	})
+
+	t.Run("missing output", func(t *testing.T) {
+		_, err := renderStepContainer("${{ outputs.imageName }}", &template.StepContext{})
+		require.Error(t, err)
+	})
+}

internal/batches/service/service.go

@@ -295,10 +295,19 @@ func (svc *Service) EnsureDockerImages(
 	parallelism int,
 	progress func(done, total int),
 ) (map[string]docker.Image, error) {
-	// Figure out the image names used in the batch spec.
+	// Figure out the concrete image names used in the batch spec. Images that
+	// still depend on runtime values, such as outputs from earlier steps, are
+	// resolved and pulled just-in-time by the executor.
 	names := map[string]struct{}{}
 	for i := range steps {
-		names[steps[i].Container] = struct{}{}
+		isStatic, name, err := templatelib.IsStaticString(steps[i].Container, &templatelib.StepContext{})
+		if err != nil {
+			return nil, err
+		}
+		if !isStatic {
+			continue
+		}
+		names[name] = struct{}{}
 	}
 
 	total := len(names)

internal/batches/service/service_test.go

@@ -113,8 +113,9 @@ func TestEnsureDockerImages(t *testing.T) {
 			}
 
 			for name, steps := range map[string][]batcheslib.Step{
-				"single step":    {{Container: "image"}},
-				"multiple steps": {{Container: "image"}, {Container: "image"}},
+				"single step":      {{Container: "image"}},
+				"multiple steps":   {{Container: "image"}, {Container: "image"}},
+				"dynamic deferred": {{Container: "${{ outputs.imageName }}"}, {Container: "image"}},
 			} {
 				t.Run(name, func(t *testing.T) {
 					for _, parallelism := range parallelCases {
@@ -265,6 +266,35 @@ some-new-field: Foo bar
 `,
 			expectedErr: errors.New("parsing batch spec: Additional property some-new-field is not allowed"),
 		},
+		{
+			name: "step image alias",
+			rawSpec: `
+name: test-spec
+description: A test spec
+steps:
+  - run: echo hi
+    image: alpine:3
+changesetTemplate:
+  title: Test
+  body: Test
+  branch: test
+  commit:
+    message: Test
+`,
+			expectedSpec: &batcheslib.BatchSpec{
+				Name:        "test-spec",
+				Description: "A test spec",
+				Steps: []batcheslib.Step{
+					{Run: "echo hi", Container: "alpine:3", Image: "alpine:3"},
+				},
+				ChangesetTemplate: &batcheslib.ChangesetTemplate{
+					Title:  "Test",
+					Body:   "Test",
+					Branch: "test",
+					Commit: batcheslib.ExpandedGitCommitDescription{Message: "Test"},
+				},
+			},
+		},
 		{
 			name: "supported version",
 			rawSpec: `

lib/batches/batch_spec.go

@@ -1,6 +1,7 @@
 package batches
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -92,13 +93,42 @@ func (oqor *OnQueryOrRepository) GetBranches() ([]string, error) {
 type Step struct {
 	Run       string            `json:"run,omitempty" yaml:"run"`
 	Container string            `json:"container,omitempty" yaml:"container"`
+	Image     string            `json:"image,omitempty" yaml:"image,omitempty"`
 	Env       env.Environment   `json:"env" yaml:"env"`
 	Files     map[string]string `json:"files,omitempty" yaml:"files,omitempty"`
 	Outputs   Outputs           `json:"outputs,omitempty" yaml:"outputs,omitempty"`
 	Mount     []Mount           `json:"mount,omitempty" yaml:"mount,omitempty"`
 	If        any               `json:"if,omitempty" yaml:"if,omitempty"`
 }
 
+func (s *Step) normalizeImageAlias() {
+	if s.Container == "" && s.Image != "" {
+		s.Container = s.Image
+	}
+}
+
+func (s *Step) UnmarshalJSON(data []byte) error {
+	type step Step
+	var decoded step
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return err
+	}
+	*s = Step(decoded)
+	s.normalizeImageAlias()
+	return nil
+}
+
+func (s *Step) UnmarshalYAML(unmarshal func(any) error) error {
+	type step Step
+	var decoded step
+	if err := unmarshal(&decoded); err != nil {
+		return err
+	}
+	*s = Step(decoded)
+	s.normalizeImageAlias()
+	return nil
+}
+
 func (s *Step) IfCondition() string {
 	switch v := s.If.(type) {
 	case bool:

lib/batches/schema/batch_spec_stringdata.go

@@ -42,6 +42,26 @@ func IsStaticBool(input string, ctx *StepContext) (isStatic bool, boolVal bool,
 	return true, isTrueOutput(t.Tree.Root), nil
 }
 
+// IsStaticString parses the input as a text/template and attempts to evaluate it
+// with only the information currently available in StepContext. If any template
+// actions remain after partial evaluation, the first return value is false.
+func IsStaticString(input string, ctx *StepContext) (isStatic bool, value string, err error) {
+	t, err := parseAndPartialEval(input, ctx)
+	if err != nil {
+		return false, "", err
+	}
+
+	var out bytes.Buffer
+	for _, n := range t.Tree.Root.Nodes {
+		if n.Type() != parse.NodeText {
+			return false, "", nil
+		}
+		out.WriteString(n.String())
+	}
+
+	return true, out.String(), nil
+}
+
 // parseAndPartialEval parses input as a text/template and then attempts to
 // partially evaluate the parts of the template it can evaluate ahead of time
 // (meaning: before we've executed any batch spec steps and have a full