From 90ea691e72c6e83ed0e8861bb1187832a69b333d Mon Sep 17 00:00:00 2001 From: Daniel Hobley Date: Sat, 27 May 2023 04:47:23 +0200 Subject: [PATCH] feat: allow for modifying `var-run` mount maximum size limit (#2624) --- TROUBLESHOOTING.md | 24 +++++++++++++++++++ .../v1alpha1/runner_types.go | 2 ++ .../v1alpha1/zz_generated.deepcopy.go | 5 ++++ ...ions.summerwind.dev_runnerdeployments.yaml | 6 +++++ ...ions.summerwind.dev_runnerreplicasets.yaml | 6 +++++ .../crds/actions.summerwind.dev_runners.yaml | 6 +++++ .../actions.summerwind.dev_runnersets.yaml | 6 +++++ ...ions.summerwind.dev_runnerdeployments.yaml | 6 +++++ ...ions.summerwind.dev_runnerreplicasets.yaml | 6 +++++ .../bases/actions.summerwind.dev_runners.yaml | 6 +++++ .../actions.summerwind.dev_runnersets.yaml | 6 +++++ .../runner_controller.go | 9 +++++-- 12 files changed, 86 insertions(+), 2 deletions(-) diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 2e5c92c..3807005 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -304,3 +304,27 @@ If you noticed that it takes several minutes for sidecar dind container to be cr **Solution** The solution is to switch to using faster storage, if you are experiencing this issue you are probably using HDD storage. Switching to SSD storage fixed the problem in my case. Most cloud providers have a list of storage options to use just pick something faster that your current disk, for on prem clusters you will need to invest in some SSDs. + +### Dockerd no space left on device + +**Problem** + +If you are running many containers on your runner you might encounter an issue where docker daemon is unable to start new containers and you see error `no space left on device`. + +**Solution** + +Add a `dockerVarRunVolumeSizeLimit` key in your runner's spec with a higher size limit (the default is 1M) For instance: + +```yaml +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: github-runner + namespace: github-system +spec: + replicas: 6 + template: + spec: + dockerVarRunVolumeSizeLimit: 50M + env: [] +``` \ No newline at end of file diff --git a/apis/actions.summerwind.net/v1alpha1/runner_types.go b/apis/actions.summerwind.net/v1alpha1/runner_types.go index 7986efa..ca62238 100644 --- a/apis/actions.summerwind.net/v1alpha1/runner_types.go +++ b/apis/actions.summerwind.net/v1alpha1/runner_types.go @@ -70,6 +70,8 @@ type RunnerConfig struct { // +optional DockerRegistryMirror *string `json:"dockerRegistryMirror,omitempty"` // +optional + DockerVarRunVolumeSizeLimit *resource.Quantity `json:"dockerVarRunVolumeSizeLimit,omitempty"` + // +optional VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"` // +optional VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"` diff --git a/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go b/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go index bc450b7..ac21ce4 100644 --- a/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go +++ b/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go @@ -436,6 +436,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) { *out = new(string) **out = **in } + if in.DockerVarRunVolumeSizeLimit != nil { + in, out := &in.DockerVarRunVolumeSizeLimit, &out.DockerVarRunVolumeSizeLimit + x := (*in).DeepCopy() + *out = &x + } if in.VolumeSizeLimit != nil { in, out := &in.VolumeSizeLimit, &out.VolumeSizeLimit x := (*in).DeepCopy() diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml index cb5f54a..4291b6b 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml @@ -1497,6 +1497,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml index a2e6aba..877bd51 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml @@ -1479,6 +1479,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml index beaea51..c688cc6 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml @@ -1432,6 +1432,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml index 7cdb815..1140928 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml @@ -55,6 +55,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerdWithinRunnerContainer: type: boolean effectiveTime: diff --git a/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml b/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml index cb5f54a..4291b6b 100644 --- a/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml @@ -1497,6 +1497,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml b/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml index a2e6aba..877bd51 100644 --- a/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml @@ -1479,6 +1479,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runners.yaml b/config/crd/bases/actions.summerwind.dev_runners.yaml index beaea51..c688cc6 100644 --- a/config/crd/bases/actions.summerwind.dev_runners.yaml +++ b/config/crd/bases/actions.summerwind.dev_runners.yaml @@ -1432,6 +1432,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runnersets.yaml b/config/crd/bases/actions.summerwind.dev_runnersets.yaml index 7cdb815..1140928 100644 --- a/config/crd/bases/actions.summerwind.dev_runnersets.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnersets.yaml @@ -55,6 +55,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerdWithinRunnerContainer: type: boolean effectiveTime: diff --git a/controllers/actions.summerwind.net/runner_controller.go b/controllers/actions.summerwind.net/runner_controller.go index 574d08a..a711fd8 100644 --- a/controllers/actions.summerwind.net/runner_controller.go +++ b/controllers/actions.summerwind.net/runner_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "k8s.io/apimachinery/pkg/api/resource" "reflect" "strconv" "strings" @@ -30,7 +31,6 @@ import ( "github.com/go-logr/logr" kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -810,6 +810,11 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru dockerRegistryMirror = *runnerSpec.DockerRegistryMirror } + if runnerSpec.DockerVarRunVolumeSizeLimit == nil { + runnerSpec.DockerVarRunVolumeSizeLimit = resource.NewScaledQuantity(1, resource.Mega) + + } + // Be aware some of the environment variables are used // in the runner entrypoint script env := []corev1.EnvVar{ @@ -1080,7 +1085,7 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: corev1.StorageMediumMemory, - SizeLimit: resource.NewScaledQuantity(1, resource.Mega), + SizeLimit: runnerSpec.DockerVarRunVolumeSizeLimit, }, }, },