323 lines
8.4 KiB
Go
323 lines
8.4 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bufio"
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"os/exec"
|
||
|
|
"path/filepath"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"sync"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/google/go-github/v79/github"
|
||
|
|
"github.com/stretchr/testify/assert"
|
||
|
|
"github.com/stretchr/testify/require"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestE2E(t *testing.T) {
|
||
|
|
if os.Getenv("E2E") != "true" {
|
||
|
|
t.Skip("Skipping E2E test; set E2E=true to run")
|
||
|
|
}
|
||
|
|
|
||
|
|
configURL := mustGetEnv(t, "E2E_SCALESET_URL")
|
||
|
|
name := mustGetEnv(t, "E2E_SCALESET_NAME")
|
||
|
|
|
||
|
|
workflowEnv := mustE2EWorkflowEnv(t, name)
|
||
|
|
runArgs := mustE2ECommandArgs(t, configURL, name)
|
||
|
|
|
||
|
|
tempDir, err := os.MkdirTemp("", "e2e-dockerscaleset-")
|
||
|
|
require.NoError(t, err, "Failed to create temp dir")
|
||
|
|
defer os.RemoveAll(tempDir)
|
||
|
|
|
||
|
|
binaryPath := filepath.Join(tempDir, "dockerscaleset")
|
||
|
|
|
||
|
|
// Build the dockerscaleset binary in temp dir
|
||
|
|
{
|
||
|
|
cmd := exec.Command("go", "build", "-o", binaryPath, ".")
|
||
|
|
output, err := cmd.CombinedOutput()
|
||
|
|
require.NoError(t, err, "Failed to build dockerscaleset: %s", output)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Fatal channel
|
||
|
|
testErrCh := make(chan error, 2)
|
||
|
|
|
||
|
|
runCmd := exec.Command(binaryPath, runArgs...)
|
||
|
|
stdout, err := runCmd.StdoutPipe()
|
||
|
|
runCmd.Stderr = os.Stderr
|
||
|
|
require.NoError(t, err, "Failed to get stdout pipe")
|
||
|
|
err = runCmd.Start()
|
||
|
|
require.NoError(t, err, "Failed to start dockerscaleset")
|
||
|
|
|
||
|
|
// Command exit error
|
||
|
|
cmdCh := make(chan error, 1)
|
||
|
|
t.Cleanup(func() {
|
||
|
|
_ = runCmd.Process.Signal(os.Interrupt)
|
||
|
|
<-cmdCh
|
||
|
|
})
|
||
|
|
|
||
|
|
// Wait for log line
|
||
|
|
waitCh := make(chan struct{}, 1)
|
||
|
|
|
||
|
|
var (
|
||
|
|
bufMu sync.Mutex
|
||
|
|
buf bytes.Buffer
|
||
|
|
)
|
||
|
|
go func() {
|
||
|
|
scanner := bufio.NewScanner(stdout)
|
||
|
|
for scanner.Scan() {
|
||
|
|
line := scanner.Text()
|
||
|
|
bufMu.Lock()
|
||
|
|
buf.WriteString(line + "\n")
|
||
|
|
bufMu.Unlock()
|
||
|
|
if strings.Contains(line, "Getting next message") {
|
||
|
|
close(waitCh)
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := scanner.Err(); err != nil {
|
||
|
|
testErrCh <- fmt.Errorf("error reading dockerscaleset stdout: %w", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
cmdCh <- runCmd.Wait()
|
||
|
|
close(cmdCh)
|
||
|
|
}()
|
||
|
|
|
||
|
|
runID, err := workflowEnv.triggerWorkflowDispatch(t, t.Context())
|
||
|
|
require.NoError(t, err, "Failed to trigger workflow")
|
||
|
|
|
||
|
|
statusCh := make(chan *WorkflowRun, 1)
|
||
|
|
go func() {
|
||
|
|
select {
|
||
|
|
case <-waitCh:
|
||
|
|
case <-time.After(30 * time.Second):
|
||
|
|
bufMu.Lock()
|
||
|
|
logs := buf.String()
|
||
|
|
bufMu.Unlock()
|
||
|
|
testErrCh <- fmt.Errorf("timeout waiting for dockerscaleset to be ready; logs:\n%s", logs)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
status, err := workflowEnv.waitForWorkflowCompletion(t, t.Context(), runID, 10*time.Minute)
|
||
|
|
if err != nil {
|
||
|
|
testErrCh <- fmt.Errorf("failed to wait for workflow completion: %w", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
statusCh <- status
|
||
|
|
}()
|
||
|
|
|
||
|
|
select {
|
||
|
|
case err := <-cmdCh:
|
||
|
|
select {
|
||
|
|
case status := <-statusCh:
|
||
|
|
assert.Equal(t, "completed", status.Status)
|
||
|
|
assert.Equal(t, "success", status.Conclusion)
|
||
|
|
case <-time.After(30 * time.Second):
|
||
|
|
bufMu.Lock()
|
||
|
|
logs := buf.String()
|
||
|
|
bufMu.Unlock()
|
||
|
|
t.Fatalf("Timeout waiting for workflow status after dockerscaleset exited\nexit: %v\nlogs:%s\n", err, logs)
|
||
|
|
}
|
||
|
|
case status := <-statusCh:
|
||
|
|
assert.NotNil(t, status, "WorkflowRun status is nil")
|
||
|
|
assert.Equal(t, "completed", status.Status)
|
||
|
|
assert.Equal(t, "success", status.Conclusion)
|
||
|
|
return
|
||
|
|
case err := <-testErrCh:
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type e2eWorkflowEnv struct {
|
||
|
|
targetOrg string
|
||
|
|
targetRepo string
|
||
|
|
targetFile string
|
||
|
|
|
||
|
|
scalesetName string
|
||
|
|
client *github.Client
|
||
|
|
}
|
||
|
|
|
||
|
|
func mustE2EWorkflowEnv(t *testing.T, scalesetName string) *e2eWorkflowEnv {
|
||
|
|
return &e2eWorkflowEnv{
|
||
|
|
targetOrg: mustGetEnv(t, "E2E_WORKFLOW_TARGET_ORG"),
|
||
|
|
targetRepo: mustGetEnv(t, "E2E_WORKFLOW_TARGET_REPO"),
|
||
|
|
targetFile: mustGetEnv(t, "E2E_WORKFLOW_TARGET_FILE"),
|
||
|
|
scalesetName: scalesetName,
|
||
|
|
client: github.NewClient(nil).WithAuthToken(mustGetEnv(t, "E2E_WORKFLOW_GITHUB_TOKEN")),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func mustE2ECommandArgs(t *testing.T, configURL, name string) []string {
|
||
|
|
args := []string{
|
||
|
|
"--url", configURL,
|
||
|
|
"--name", name,
|
||
|
|
"--log-level", "debug",
|
||
|
|
}
|
||
|
|
|
||
|
|
// GitHub App credentials
|
||
|
|
var (
|
||
|
|
clientID string
|
||
|
|
installationID int
|
||
|
|
privateKeyPath string
|
||
|
|
)
|
||
|
|
|
||
|
|
// GitHub token
|
||
|
|
var token string
|
||
|
|
|
||
|
|
clientID = os.Getenv("E2E_SCALESET_GITHUB_APP_CLIENT_ID")
|
||
|
|
installationIDStr := os.Getenv("E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
|
||
|
|
privateKeyPath = os.Getenv("E2E_SCALESET_GITHUB_APP_PRIVATE_KEY_PATH")
|
||
|
|
|
||
|
|
if clientID != "" && installationIDStr != "" && privateKeyPath != "" {
|
||
|
|
id, err := strconv.Atoi(installationIDStr)
|
||
|
|
require.NoError(t, err, "Invalid E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
|
||
|
|
installationID = id
|
||
|
|
args = append(args,
|
||
|
|
"--app-client-id", clientID,
|
||
|
|
"--app-installation-id", fmt.Sprintf("%d", installationID),
|
||
|
|
"--app-private-key", privateKeyPath,
|
||
|
|
)
|
||
|
|
} else {
|
||
|
|
token = os.Getenv("E2E_SCALESET_GITHUB_TOKEN")
|
||
|
|
require.NotEmpty(t, token, "E2E_SCALESET_GITHUB_TOKEN must be set if GitHub App credentials are not provided")
|
||
|
|
args = append(args,
|
||
|
|
"--token", token,
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
runnerGroup := os.Getenv("E2E_SCALESET_RUNNER_GROUP")
|
||
|
|
if runnerGroup != "" {
|
||
|
|
args = append(args,
|
||
|
|
"--runner-group", runnerGroup,
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
minRunners := 0
|
||
|
|
if minRunnersStr := os.Getenv("E2E_SCALESET_MIN_RUNNERS"); minRunnersStr != "" {
|
||
|
|
m, err := strconv.Atoi(minRunnersStr)
|
||
|
|
require.NoError(t, err, "Invalid E2E_SCALESET_MIN_RUNNERS")
|
||
|
|
minRunners = m
|
||
|
|
require.GreaterOrEqual(t, minRunners, 0, "E2E_SCALESET_MIN_RUNNERS must be >= 0")
|
||
|
|
}
|
||
|
|
|
||
|
|
maxRunners := 10
|
||
|
|
if maxRunnersStr := os.Getenv("E2E_SCALESET_MAX_RUNNERS"); maxRunnersStr != "" {
|
||
|
|
m, err := strconv.Atoi(maxRunnersStr)
|
||
|
|
require.NoError(t, err, "Invalid E2E_SCALESET_MAX_RUNNERS")
|
||
|
|
maxRunners = m
|
||
|
|
require.GreaterOrEqual(t, maxRunners, 0, "E2E_SCALESET_MAX_RUNNERS must be >= 0")
|
||
|
|
}
|
||
|
|
|
||
|
|
require.GreaterOrEqual(t, maxRunners, minRunners, "E2E_SCALESET_MAX_RUNNERS must be >= E2E_SCALESET_MIN_RUNNERS")
|
||
|
|
|
||
|
|
args = append(args,
|
||
|
|
"--min-runners", strconv.Itoa(minRunners),
|
||
|
|
"--max-runners", strconv.Itoa(maxRunners),
|
||
|
|
)
|
||
|
|
|
||
|
|
return args
|
||
|
|
}
|
||
|
|
|
||
|
|
type WorkflowRun struct {
|
||
|
|
ID int `json:"id"`
|
||
|
|
Status string `json:"status"`
|
||
|
|
Conclusion string `json:"conclusion"`
|
||
|
|
CreatedAt string `json:"created_at"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func (env *e2eWorkflowEnv) triggerWorkflowDispatch(t *testing.T, ctx context.Context) (int, error) {
|
||
|
|
dispatchTime := time.Now().UTC()
|
||
|
|
|
||
|
|
resp, err := env.client.Actions.CreateWorkflowDispatchEventByFileName(
|
||
|
|
ctx,
|
||
|
|
env.targetOrg,
|
||
|
|
env.targetRepo,
|
||
|
|
env.targetFile,
|
||
|
|
github.CreateWorkflowDispatchEventRequest{
|
||
|
|
Ref: "main",
|
||
|
|
Inputs: map[string]any{
|
||
|
|
"scaleset_name": env.scalesetName,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
)
|
||
|
|
require.NoError(t, err, "Failed to create workflow dispatch")
|
||
|
|
require.Equal(t, 204, resp.StatusCode, "Unexpected status code from workflow dispatch")
|
||
|
|
|
||
|
|
// Wait a bit for the run to be created
|
||
|
|
time.Sleep(10 * time.Second)
|
||
|
|
|
||
|
|
// List runs with event=workflow_dispatch and since=dispatchTime
|
||
|
|
opts := &github.ListWorkflowRunsOptions{
|
||
|
|
Event: "workflow_dispatch",
|
||
|
|
Created: ">=" + dispatchTime.Format(time.RFC3339),
|
||
|
|
ListOptions: github.ListOptions{
|
||
|
|
PerPage: 10,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
runs, _, err := env.client.Actions.ListWorkflowRunsByFileName(
|
||
|
|
t.Context(),
|
||
|
|
env.targetOrg,
|
||
|
|
env.targetRepo,
|
||
|
|
env.targetFile,
|
||
|
|
opts,
|
||
|
|
)
|
||
|
|
require.NoError(t, err, "Failed to list workflow runs")
|
||
|
|
require.Greater(t, len(runs.WorkflowRuns), 0, "No workflow runs found after dispatch")
|
||
|
|
|
||
|
|
// Sort by created_at desc, take the first (most recent)
|
||
|
|
var latestRun *github.WorkflowRun
|
||
|
|
var latestTime time.Time
|
||
|
|
for _, run := range runs.WorkflowRuns {
|
||
|
|
createdAt := run.CreatedAt.Time
|
||
|
|
if createdAt.After(latestTime) {
|
||
|
|
latestTime = createdAt
|
||
|
|
latestRun = run
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if latestRun == nil {
|
||
|
|
return 0, fmt.Errorf("no workflow runs found after dispatch")
|
||
|
|
}
|
||
|
|
|
||
|
|
return int(latestRun.GetID()), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (env *e2eWorkflowEnv) waitForWorkflowCompletion(t *testing.T, ctx context.Context, runID int, timeout time.Duration) (*WorkflowRun, error) {
|
||
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||
|
|
defer cancel()
|
||
|
|
|
||
|
|
ticker := time.NewTicker(5 * time.Second)
|
||
|
|
defer ticker.Stop()
|
||
|
|
|
||
|
|
for {
|
||
|
|
select {
|
||
|
|
case <-ctx.Done():
|
||
|
|
return nil, ctx.Err()
|
||
|
|
case <-ticker.C:
|
||
|
|
run, _, err := env.client.Actions.GetWorkflowRunByID(ctx, env.targetOrg, env.targetRepo, int64(runID))
|
||
|
|
require.NoError(t, err, "Failed to get workflow run by ID")
|
||
|
|
|
||
|
|
if run.GetStatus() == "completed" {
|
||
|
|
return &WorkflowRun{
|
||
|
|
ID: int(run.GetID()),
|
||
|
|
Status: run.GetStatus(),
|
||
|
|
Conclusion: run.GetConclusion(),
|
||
|
|
CreatedAt: run.GetCreatedAt().Format(time.RFC3339),
|
||
|
|
}, nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func mustGetEnv(t *testing.T, key string) string {
|
||
|
|
value := os.Getenv(key)
|
||
|
|
if value == "" {
|
||
|
|
t.Fatalf("Environment variable %s not set", key)
|
||
|
|
}
|
||
|
|
return value
|
||
|
|
}
|