Implement dockerscaleset e2e tests (#25)
* wip: setting up e2e tests * rebase
This commit is contained in:
Nikola Jokic
@@ -0,0 +1,322 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-github/v79/github"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestE2E(t *testing.T) {
|
||||
if os.Getenv("E2E") != "true" {
|
||||
t.Skip("Skipping E2E test; set E2E=true to run")
|
||||
}
|
||||
|
||||
configURL := mustGetEnv(t, "E2E_SCALESET_URL")
|
||||
name := mustGetEnv(t, "E2E_SCALESET_NAME")
|
||||
|
||||
workflowEnv := mustE2EWorkflowEnv(t, name)
|
||||
runArgs := mustE2ECommandArgs(t, configURL, name)
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "e2e-dockerscaleset-")
|
||||
require.NoError(t, err, "Failed to create temp dir")
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
binaryPath := filepath.Join(tempDir, "dockerscaleset")
|
||||
|
||||
// Build the dockerscaleset binary in temp dir
|
||||
{
|
||||
cmd := exec.Command("go", "build", "-o", binaryPath, ".")
|
||||
output, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, "Failed to build dockerscaleset: %s", output)
|
||||
}
|
||||
|
||||
// Fatal channel
|
||||
testErrCh := make(chan error, 2)
|
||||
|
||||
runCmd := exec.Command(binaryPath, runArgs...)
|
||||
stdout, err := runCmd.StdoutPipe()
|
||||
runCmd.Stderr = os.Stderr
|
||||
require.NoError(t, err, "Failed to get stdout pipe")
|
||||
err = runCmd.Start()
|
||||
require.NoError(t, err, "Failed to start dockerscaleset")
|
||||
|
||||
// Command exit error
|
||||
cmdCh := make(chan error, 1)
|
||||
t.Cleanup(func() {
|
||||
_ = runCmd.Process.Signal(os.Interrupt)
|
||||
<-cmdCh
|
||||
})
|
||||
|
||||
// Wait for log line
|
||||
waitCh := make(chan struct{}, 1)
|
||||
|
||||
var (
|
||||
bufMu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
)
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
bufMu.Lock()
|
||||
buf.WriteString(line + "\n")
|
||||
bufMu.Unlock()
|
||||
if strings.Contains(line, "Getting next message") {
|
||||
close(waitCh)
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
testErrCh <- fmt.Errorf("error reading dockerscaleset stdout: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
cmdCh <- runCmd.Wait()
|
||||
close(cmdCh)
|
||||
}()
|
||||
|
||||
runID, err := workflowEnv.triggerWorkflowDispatch(t, t.Context())
|
||||
require.NoError(t, err, "Failed to trigger workflow")
|
||||
|
||||
statusCh := make(chan *WorkflowRun, 1)
|
||||
go func() {
|
||||
select {
|
||||
case <-waitCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
bufMu.Lock()
|
||||
logs := buf.String()
|
||||
bufMu.Unlock()
|
||||
testErrCh <- fmt.Errorf("timeout waiting for dockerscaleset to be ready; logs:\n%s", logs)
|
||||
return
|
||||
}
|
||||
status, err := workflowEnv.waitForWorkflowCompletion(t, t.Context(), runID, 10*time.Minute)
|
||||
if err != nil {
|
||||
testErrCh <- fmt.Errorf("failed to wait for workflow completion: %w", err)
|
||||
return
|
||||
}
|
||||
statusCh <- status
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-cmdCh:
|
||||
select {
|
||||
case status := <-statusCh:
|
||||
assert.Equal(t, "completed", status.Status)
|
||||
assert.Equal(t, "success", status.Conclusion)
|
||||
case <-time.After(30 * time.Second):
|
||||
bufMu.Lock()
|
||||
logs := buf.String()
|
||||
bufMu.Unlock()
|
||||
t.Fatalf("Timeout waiting for workflow status after dockerscaleset exited\nexit: %v\nlogs:%s\n", err, logs)
|
||||
}
|
||||
case status := <-statusCh:
|
||||
assert.NotNil(t, status, "WorkflowRun status is nil")
|
||||
assert.Equal(t, "completed", status.Status)
|
||||
assert.Equal(t, "success", status.Conclusion)
|
||||
return
|
||||
case err := <-testErrCh:
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type e2eWorkflowEnv struct {
|
||||
targetOrg string
|
||||
targetRepo string
|
||||
targetFile string
|
||||
|
||||
scalesetName string
|
||||
client *github.Client
|
||||
}
|
||||
|
||||
func mustE2EWorkflowEnv(t *testing.T, scalesetName string) *e2eWorkflowEnv {
|
||||
return &e2eWorkflowEnv{
|
||||
targetOrg: mustGetEnv(t, "E2E_WORKFLOW_TARGET_ORG"),
|
||||
targetRepo: mustGetEnv(t, "E2E_WORKFLOW_TARGET_REPO"),
|
||||
targetFile: mustGetEnv(t, "E2E_WORKFLOW_TARGET_FILE"),
|
||||
scalesetName: scalesetName,
|
||||
client: github.NewClient(nil).WithAuthToken(mustGetEnv(t, "E2E_WORKFLOW_GITHUB_TOKEN")),
|
||||
}
|
||||
}
|
||||
|
||||
func mustE2ECommandArgs(t *testing.T, configURL, name string) []string {
|
||||
args := []string{
|
||||
"--url", configURL,
|
||||
"--name", name,
|
||||
"--log-level", "debug",
|
||||
}
|
||||
|
||||
// GitHub App credentials
|
||||
var (
|
||||
clientID string
|
||||
installationID int
|
||||
privateKeyPath string
|
||||
)
|
||||
|
||||
// GitHub token
|
||||
var token string
|
||||
|
||||
clientID = os.Getenv("E2E_SCALESET_GITHUB_APP_CLIENT_ID")
|
||||
installationIDStr := os.Getenv("E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
|
||||
privateKeyPath = os.Getenv("E2E_SCALESET_GITHUB_APP_PRIVATE_KEY_PATH")
|
||||
|
||||
if clientID != "" && installationIDStr != "" && privateKeyPath != "" {
|
||||
id, err := strconv.Atoi(installationIDStr)
|
||||
require.NoError(t, err, "Invalid E2E_SCALESET_GITHUB_APP_INSTALLATION_ID")
|
||||
installationID = id
|
||||
args = append(args,
|
||||
"--app-client-id", clientID,
|
||||
"--app-installation-id", fmt.Sprintf("%d", installationID),
|
||||
"--app-private-key", privateKeyPath,
|
||||
)
|
||||
} else {
|
||||
token = os.Getenv("E2E_SCALESET_GITHUB_TOKEN")
|
||||
require.NotEmpty(t, token, "E2E_SCALESET_GITHUB_TOKEN must be set if GitHub App credentials are not provided")
|
||||
args = append(args,
|
||||
"--token", token,
|
||||
)
|
||||
}
|
||||
|
||||
runnerGroup := os.Getenv("E2E_SCALESET_RUNNER_GROUP")
|
||||
if runnerGroup != "" {
|
||||
args = append(args,
|
||||
"--runner-group", runnerGroup,
|
||||
)
|
||||
}
|
||||
|
||||
minRunners := 0
|
||||
if minRunnersStr := os.Getenv("E2E_SCALESET_MIN_RUNNERS"); minRunnersStr != "" {
|
||||
m, err := strconv.Atoi(minRunnersStr)
|
||||
require.NoError(t, err, "Invalid E2E_SCALESET_MIN_RUNNERS")
|
||||
minRunners = m
|
||||
require.GreaterOrEqual(t, minRunners, 0, "E2E_SCALESET_MIN_RUNNERS must be >= 0")
|
||||
}
|
||||
|
||||
maxRunners := 10
|
||||
if maxRunnersStr := os.Getenv("E2E_SCALESET_MAX_RUNNERS"); maxRunnersStr != "" {
|
||||
m, err := strconv.Atoi(maxRunnersStr)
|
||||
require.NoError(t, err, "Invalid E2E_SCALESET_MAX_RUNNERS")
|
||||
maxRunners = m
|
||||
require.GreaterOrEqual(t, maxRunners, 0, "E2E_SCALESET_MAX_RUNNERS must be >= 0")
|
||||
}
|
||||
|
||||
require.GreaterOrEqual(t, maxRunners, minRunners, "E2E_SCALESET_MAX_RUNNERS must be >= E2E_SCALESET_MIN_RUNNERS")
|
||||
|
||||
args = append(args,
|
||||
"--min-runners", strconv.Itoa(minRunners),
|
||||
"--max-runners", strconv.Itoa(maxRunners),
|
||||
)
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
type WorkflowRun struct {
|
||||
ID int `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Conclusion string `json:"conclusion"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
func (env *e2eWorkflowEnv) triggerWorkflowDispatch(t *testing.T, ctx context.Context) (int, error) {
|
||||
dispatchTime := time.Now().UTC()
|
||||
|
||||
resp, err := env.client.Actions.CreateWorkflowDispatchEventByFileName(
|
||||
ctx,
|
||||
env.targetOrg,
|
||||
env.targetRepo,
|
||||
env.targetFile,
|
||||
github.CreateWorkflowDispatchEventRequest{
|
||||
Ref: "main",
|
||||
Inputs: map[string]any{
|
||||
"scaleset_name": env.scalesetName,
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err, "Failed to create workflow dispatch")
|
||||
require.Equal(t, 204, resp.StatusCode, "Unexpected status code from workflow dispatch")
|
||||
|
||||
// Wait a bit for the run to be created
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
// List runs with event=workflow_dispatch and since=dispatchTime
|
||||
opts := &github.ListWorkflowRunsOptions{
|
||||
Event: "workflow_dispatch",
|
||||
Created: ">=" + dispatchTime.Format(time.RFC3339),
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: 10,
|
||||
},
|
||||
}
|
||||
runs, _, err := env.client.Actions.ListWorkflowRunsByFileName(
|
||||
t.Context(),
|
||||
env.targetOrg,
|
||||
env.targetRepo,
|
||||
env.targetFile,
|
||||
opts,
|
||||
)
|
||||
require.NoError(t, err, "Failed to list workflow runs")
|
||||
require.Greater(t, len(runs.WorkflowRuns), 0, "No workflow runs found after dispatch")
|
||||
|
||||
// Sort by created_at desc, take the first (most recent)
|
||||
var latestRun *github.WorkflowRun
|
||||
var latestTime time.Time
|
||||
for _, run := range runs.WorkflowRuns {
|
||||
createdAt := run.CreatedAt.Time
|
||||
if createdAt.After(latestTime) {
|
||||
latestTime = createdAt
|
||||
latestRun = run
|
||||
}
|
||||
}
|
||||
|
||||
if latestRun == nil {
|
||||
return 0, fmt.Errorf("no workflow runs found after dispatch")
|
||||
}
|
||||
|
||||
return int(latestRun.GetID()), nil
|
||||
}
|
||||
|
||||
func (env *e2eWorkflowEnv) waitForWorkflowCompletion(t *testing.T, ctx context.Context, runID int, timeout time.Duration) (*WorkflowRun, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-ticker.C:
|
||||
run, _, err := env.client.Actions.GetWorkflowRunByID(ctx, env.targetOrg, env.targetRepo, int64(runID))
|
||||
require.NoError(t, err, "Failed to get workflow run by ID")
|
||||
|
||||
if run.GetStatus() == "completed" {
|
||||
return &WorkflowRun{
|
||||
ID: int(run.GetID()),
|
||||
Status: run.GetStatus(),
|
||||
Conclusion: run.GetConclusion(),
|
||||
CreatedAt: run.GetCreatedAt().Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustGetEnv(t *testing.T, key string) string {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
t.Fatalf("Environment variable %s not set", key)
|
||||
}
|
||||
return value
|
||||
}
|
||||
Reference in New Issue
Block a user