Implement dockerscaleset e2e tests (#25)

* wip: setting up e2e tests

* rebase

.github/workflows/e2e.yaml

@@ -0,0 +1,55 @@
+name: E2E
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  E2E_WORKFLOW_TARGET_ORG: "scaleset-canary"
+  E2E_WORKFLOW_TARGET_REPO: "e2e"
+
+jobs:
+  basic-e2e:
+    name: Basic E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      E2E_WORKFLOW_TARGET_FILE: "basic.yaml"
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: true
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: "go.mod"
+      - name: Get configure token
+        id: config-token
+        uses: peter-murray/workflow-application-token-action@d17e3a9a36850ea89f35db16c1067dd2b68ee343
+        with:
+          application_id: ${{ secrets.E2E_TESTS_ACCESS_CLIENT_ID }}
+          application_private_key: ${{ secrets.E2E_TESTS_ACCESS_PK }}
+          organization: ${{ env.E2E_WORKFLOW_TARGET_ORG }}
+      - name: Run simple test
+        run: |
+          git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/actions/scaleset".insteadOf "https://github.com/actions/scaleset"
+          E2E_SCALESET_NAME="basic-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))" go test
+        working-directory: examples/dockerscaleset
+        env:
+          GOPRIVATE: github.com/actions/scaleset
+          GONOSUMDB: github.com/actions/scaleset
+          E2E_SCALESET_GITHUB_TOKEN: "${{steps.config-token.outputs.token}}"
+          E2E_WORKFLOW_GITHUB_TOKEN: "${{steps.config-token.outputs.token}}"
+          E2E_SCALESET_URL: "https://github.com/scaleset-canary/e2e"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          E2E: "true"

examples/dockerscaleset/e2e_test.go

@@ -0,0 +1,322 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-github/v79/github"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestE2E(t *testing.T) {
+	if os.Getenv("E2E") != "true" {
+		t.Skip("Skipping E2E test; set E2E=true to run")
+	}
+
+	configURL := mustGetEnv(t, "E2E_SCALESET_URL")
+	name := mustGetEnv(t, "E2E_SCALESET_NAME")
+
+	workflowEnv := mustE2EWorkflowEnv(t, name)
+	runArgs := mustE2ECommandArgs(t, configURL, name)
+
+	tempDir, err := os.MkdirTemp("", "e2e-dockerscaleset-")
+	require.NoError(t, err, "Failed to create temp dir")
+	defer os.RemoveAll(tempDir)
+
+	binaryPath := filepath.Join(tempDir, "dockerscaleset")
+
+	// Build the dockerscaleset binary in temp dir
+	{
+		cmd := exec.Command("go", "build", "-o", binaryPath, ".")
+		output, err := cmd.CombinedOutput()
+		require.NoError(t, err, "Failed to build dockerscaleset: %s", output)
+	}
+
+	// Fatal channel
+	testErrCh := make(chan error, 2)
+
+	runCmd := exec.Command(binaryPath, runArgs...)
+	stdout, err := runCmd.StdoutPipe()
+	runCmd.Stderr = os.Stderr
+	require.NoError(t, err, "Failed to get stdout pipe")
+	err = runCmd.Start()
+	require.NoError(t, err, "Failed to start dockerscaleset")
+
+	// Command exit error
+	cmdCh := make(chan error, 1)
+	t.Cleanup(func() {
+		_ = runCmd.Process.Signal(os.Interrupt)
+		<-cmdCh
+	})
+
+	// Wait for log line
+	waitCh := make(chan struct{}, 1)
+
+	var (
+		bufMu sync.Mutex
+		buf   bytes.Buffer
+	)
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			line := scanner.Text()
+			bufMu.Lock()
+			buf.WriteString(line + "\n")
+			bufMu.Unlock()
+			if strings.Contains(line, "Getting next message") {
+				close(waitCh)
+				break
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			testErrCh <- fmt.Errorf("error reading dockerscaleset stdout: %w", err)
+			return
+		}
+
+		cmdCh <- runCmd.Wait()
+		close(cmdCh)
+	}()
+
+	runID, err := workflowEnv.triggerWorkflowDispatch(t, t.Context())
+	require.NoError(t, err, "Failed to trigger workflow")
+
+	statusCh := make(chan *WorkflowRun, 1)
+	go func() {
+		select {
+		case <-waitCh:
+		case <-time.After(30 * time.Second):
+			bufMu.Lock()
+			logs := buf.String()
+			bufMu.Unlock()
+			testErrCh <- fmt.Errorf("timeout waiting for dockerscaleset to be ready; logs:\n%s", logs)
+			return
+		}
+		status, err := workflowEnv.waitForWorkflowCompletion(t, t.Context(), runID, 10*time.Minute)
+		if err != nil {
+			testErrCh <- fmt.Errorf("failed to wait for workflow completion: %w", err)
+			return
+		}
+		statusCh <- status
+	}()
+
+	select {
+	case err := <-cmdCh:
+		select {
+		case status := <-statusCh:
+			assert.Equal(t, "completed", status.Status)
+			assert.Equal(t, "success", status.Conclusion)
+		case <-time.After(30 * time.Second):
+			bufMu.Lock()
+			logs := buf.String()
+			bufMu.Unlock()
+			t.Fatalf("Timeout waiting for workflow status after dockerscaleset exited\nexit: %v\nlogs:%s\n", err, logs)
+		}
+	case status := <-statusCh:
+		assert.NotNil(t, status, "WorkflowRun status is nil")
+		assert.Equal(t, "completed", status.Status)
+		assert.Equal(t, "success", status.Conclusion)
+		return
+	case err := <-testErrCh:
+		t.Fatal(err)
+	}
+}
+
+type e2eWorkflowEnv struct {
+	targetOrg  string
+	targetRepo string
+	targetFile string
+
+	scalesetName string
+	client       *github.Client
+}
+
+func mustE2EWorkflowEnv(t *testing.T, scalesetName string) *e2eWorkflowEnv {
+	return &e2eWorkflowEnv{
+		targetOrg:    mustGetEnv(t, "E2E_WORKFLOW_TARGET_ORG"),
+		targetRepo:   mustGetEnv(t, "E2E_WORKFLOW_TARGET_REPO"),
+		targetFile:   mustGetEnv(t, "E2E_WORKFLOW_TARGET_FILE"),
+		scalesetName: scalesetName,
+		client:       github.NewClient(nil).WithAuthToken(mustGetEnv(t, "E2E_WORKFLOW_GITHUB_TOKEN")),
+	}
+}
+
+func mustE2ECommandArgs(t *testing.T, configURL, name string) []string {
+	args := []string{
+		"--url", configURL,
+		"--name", name,
+		"--log-level", "debug",
+	}
+
+	// GitHub App credentials
+	var (
+		clientID       string
+		installationID int
+		privateKeyPath string
+	)
+
+	// GitHub token
+	var token string
+
+	clientID = os.Getenv("E2E_SCALESET_GITHUB_APP_CLIENT_ID")
+	installationIDStr := os.Getenv("E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
+	privateKeyPath = os.Getenv("E2E_SCALESET_GITHUB_APP_PRIVATE_KEY_PATH")
+
+	if clientID != "" && installationIDStr != "" && privateKeyPath != "" {
+		id, err := strconv.Atoi(installationIDStr)
+		require.NoError(t, err, "Invalid E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
+		installationID = id
+		args = append(args,
+			"--app-client-id", clientID,
+			"--app-installation-id", fmt.Sprintf("%d", installationID),
+			"--app-private-key", privateKeyPath,
+		)
+	} else {
+		token = os.Getenv("E2E_SCALESET_GITHUB_TOKEN")
+		require.NotEmpty(t, token, "E2E_SCALESET_GITHUB_TOKEN must be set if GitHub App credentials are not provided")
+		args = append(args,
+			"--token", token,
+		)
+	}
+
+	runnerGroup := os.Getenv("E2E_SCALESET_RUNNER_GROUP")
+	if runnerGroup != "" {
+		args = append(args,
+			"--runner-group", runnerGroup,
+		)
+	}
+
+	minRunners := 0
+	if minRunnersStr := os.Getenv("E2E_SCALESET_MIN_RUNNERS"); minRunnersStr != "" {
+		m, err := strconv.Atoi(minRunnersStr)
+		require.NoError(t, err, "Invalid E2E_SCALESET_MIN_RUNNERS")
+		minRunners = m
+		require.GreaterOrEqual(t, minRunners, 0, "E2E_SCALESET_MIN_RUNNERS must be >= 0")
+	}
+
+	maxRunners := 10
+	if maxRunnersStr := os.Getenv("E2E_SCALESET_MAX_RUNNERS"); maxRunnersStr != "" {
+		m, err := strconv.Atoi(maxRunnersStr)
+		require.NoError(t, err, "Invalid E2E_SCALESET_MAX_RUNNERS")
+		maxRunners = m
+		require.GreaterOrEqual(t, maxRunners, 0, "E2E_SCALESET_MAX_RUNNERS must be >= 0")
+	}
+
+	require.GreaterOrEqual(t, maxRunners, minRunners, "E2E_SCALESET_MAX_RUNNERS must be >= E2E_SCALESET_MIN_RUNNERS")
+
+	args = append(args,
+		"--min-runners", strconv.Itoa(minRunners),
+		"--max-runners", strconv.Itoa(maxRunners),
+	)
+
+	return args
+}
+
+type WorkflowRun struct {
+	ID         int    `json:"id"`
+	Status     string `json:"status"`
+	Conclusion string `json:"conclusion"`
+	CreatedAt  string `json:"created_at"`
+}
+
+func (env *e2eWorkflowEnv) triggerWorkflowDispatch(t *testing.T, ctx context.Context) (int, error) {
+	dispatchTime := time.Now().UTC()
+
+	resp, err := env.client.Actions.CreateWorkflowDispatchEventByFileName(
+		ctx,
+		env.targetOrg,
+		env.targetRepo,
+		env.targetFile,
+		github.CreateWorkflowDispatchEventRequest{
+			Ref: "main",
+			Inputs: map[string]any{
+				"scaleset_name": env.scalesetName,
+			},
+		},
+	)
+	require.NoError(t, err, "Failed to create workflow dispatch")
+	require.Equal(t, 204, resp.StatusCode, "Unexpected status code from workflow dispatch")
+
+	// Wait a bit for the run to be created
+	time.Sleep(10 * time.Second)
+
+	// List runs with event=workflow_dispatch and since=dispatchTime
+	opts := &github.ListWorkflowRunsOptions{
+		Event:   "workflow_dispatch",
+		Created: ">=" + dispatchTime.Format(time.RFC3339),
+		ListOptions: github.ListOptions{
+			PerPage: 10,
+		},
+	}
+	runs, _, err := env.client.Actions.ListWorkflowRunsByFileName(
+		t.Context(),
+		env.targetOrg,
+		env.targetRepo,
+		env.targetFile,
+		opts,
+	)
+	require.NoError(t, err, "Failed to list workflow runs")
+	require.Greater(t, len(runs.WorkflowRuns), 0, "No workflow runs found after dispatch")
+
+	// Sort by created_at desc, take the first (most recent)
+	var latestRun *github.WorkflowRun
+	var latestTime time.Time
+	for _, run := range runs.WorkflowRuns {
+		createdAt := run.CreatedAt.Time
+		if createdAt.After(latestTime) {
+			latestTime = createdAt
+			latestRun = run
+		}
+	}
+
+	if latestRun == nil {
+		return 0, fmt.Errorf("no workflow runs found after dispatch")
+	}
+
+	return int(latestRun.GetID()), nil
+}
+
+func (env *e2eWorkflowEnv) waitForWorkflowCompletion(t *testing.T, ctx context.Context, runID int, timeout time.Duration) (*WorkflowRun, error) {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-ticker.C:
+			run, _, err := env.client.Actions.GetWorkflowRunByID(ctx, env.targetOrg, env.targetRepo, int64(runID))
+			require.NoError(t, err, "Failed to get workflow run by ID")
+
+			if run.GetStatus() == "completed" {
+				return &WorkflowRun{
+					ID:         int(run.GetID()),
+					Status:     run.GetStatus(),
+					Conclusion: run.GetConclusion(),
+					CreatedAt:  run.GetCreatedAt().Format(time.RFC3339),
+				}, nil
+			}
+		}
+	}
+}
+
+func mustGetEnv(t *testing.T, key string) string {
+	value := os.Getenv(key)
+	if value == "" {
+		t.Fatalf("Environment variable %s not set", key)
+	}
+	return value
+}

go.mod

@@ -24,6 +24,8 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/go-github/v79 v79.0.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect

go.sum

@@ -32,8 +32,13 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
 github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-github/v79 v79.0.0 h1:MdodQojuFPBhmtwHiBcIGLw/e/wei2PvFX9ndxK0X4Y=
+github.com/google/go-github/v79 v79.0.0/go.mod h1:OAFbNhq7fQwohojb06iIIQAB9CBGYLq999myfUFnrS4=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA=
@@ -120,6 +125,7 @@ golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
 golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0=
 google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4=