Add userdata support for runner VMs

README.md

@@ -67,6 +67,8 @@ The Orka GitHub runner requires the following environment variabales to be confi
 * `ORKA_VM_USERNAME`: Specifies the username for the deployed VMs. If no value is provided, it defaults to admin.
 * `ORKA_VM_PASSWORD`: Specifies the password for the deployed VMs. If no value is provided, it defaults to admin.
 * `ORKA_VM_METADATA`: Specifies custom VM metadata passed to the VM. Must be formatted as key=value comma separated pairs.
+* `ORKA_VM_USERDATA`: Specifies a userdata bootstrap script executed as root inside each runner VM on its first boot, before the GitHub runner starts. Use it to prepare the VM (install tools, enroll in MDM, register services) without baking the steps into the VM image. The script runs in the background and never delays or fails the VM boot; its output is written to `/var/log/orka/userdata.log` inside the VM. The script content is kept out of the integration's logs. Requires Orka 3.7.0 or later (both the cluster and the `orka3` CLI) and an Apple Silicon VM.
+* `ORKA_VM_USERDATA_FILE_PATH`: Specifies a path to a file containing the userdata bootstrap script. Ignored when `ORKA_VM_USERDATA` is set.
 * `ORKA_ENABLE_NODE_IP_MAPPING`: Specifies whether to enable the mapping of Orka node IPs to external IPs.
 * `ORKA_NODE_IP_MAPPING`: Defines the mapping of Orka node internal IPs to external host IPs.
 * `RUNNERS`: A JSON array containing configuration details of the GitHub runner scale set that will be created. Currently only one runner is supported. See [here](#how-to-use-multiple-runners) for how to use multiple runners. Example usage: `RUNNERS='[{"name":"my-github-runner", "id": 1}]'`. The `name` field should match the value specified in the `runs-on` field in the Actions workflow. The `id` field should be used to differentiate runners with GitHub. We default to `1` if it is not defined. See an example [here](./examples/ci.yml).

examples/.env

@@ -98,6 +98,19 @@ LOG_LEVEL="debug"
 # Should be formatted as "key=value" comma separated list.
 ORKA_VM_METADATA="key1=value1,key2=value2"
 
+# [Optional] ORKA_VM_USERDATA specifies a bootstrap script executed as root inside each
+# runner VM on its first boot, before the GitHub runner starts. Use it to prepare the VM
+# (install tools, enroll in MDM, register services) without baking the steps into the image.
+# The script runs in the background, never delays or fails the VM boot, and its output is
+# written to /var/log/orka/userdata.log inside the VM.
+# Requires Orka 3.7.0 or later (cluster and orka3 CLI) and an Apple Silicon VM.
+# ORKA_VM_USERDATA="#!/bin/bash
+# echo hello > /Users/Shared/bootstrap-done.txt"
+
+# [Optional] ORKA_VM_USERDATA_FILE_PATH specifies a path to a file containing the userdata
+# bootstrap script. Ignored when ORKA_VM_USERDATA is set.
+# ORKA_VM_USERDATA_FILE_PATH="/path/to/bootstrap.sh"
+
 # [Optional] VM_TRACKER_INTERVAL specifies the interval at which the VM tracker will check for orphaned VMs.
 # VMs are deleted if they do not have a corresponding GitHub runner for 2 consecutive checks.
 # If not provided, it defaults to 300 seconds.

pkg/env/constants.go

@@ -1,5 +1,9 @@
 package env
 
+// maxUserdataEncodedSize is the maximum size of the base64-encoded userdata
+// script accepted by the Orka API (64KiB).
+const maxUserdataEncodedSize = 65536
+
 const (
 	GitHubAppIDEnvName             = "GITHUB_APP_ID"
 	GitHubAppInstallationIDEnvName = "GITHUB_APP_INSTALLATION_ID"
@@ -14,11 +18,13 @@ const (
 	OrkaURLEnvName   = "ORKA_URL"
 	OrkaTokenEnvName = "ORKA_TOKEN"
 
-	OrkaNamespaceEnvName  = "ORKA_NAMESPACE"
-	OrkaVMConfigEnvName   = "ORKA_VM_CONFIG"
-	OrkaVMUsernameEnvName = "ORKA_VM_USERNAME"
-	OrkaVMPasswordEnvName = "ORKA_VM_PASSWORD"
-	OrkaVMMetadataEnvName = "ORKA_VM_METADATA"
+	OrkaNamespaceEnvName          = "ORKA_NAMESPACE"
+	OrkaVMConfigEnvName           = "ORKA_VM_CONFIG"
+	OrkaVMUsernameEnvName         = "ORKA_VM_USERNAME"
+	OrkaVMPasswordEnvName         = "ORKA_VM_PASSWORD"
+	OrkaVMMetadataEnvName         = "ORKA_VM_METADATA"
+	OrkaVMUserdataEnvName         = "ORKA_VM_USERDATA"
+	OrkaVMUserdataFilePathEnvName = "ORKA_VM_USERDATA_FILE_PATH"
 
 	OrkaEnableNodeIPMappingEnvName = "ORKA_ENABLE_NODE_IP_MAPPING"
 	OrkaNodeIPMappingEnvName       = "ORKA_NODE_IP_MAPPING"

pkg/env/env.go

@@ -1,6 +1,7 @@
 package env
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -40,6 +41,7 @@ type Data struct {
 	OrkaVMUsername string
 	OrkaVMPassword string
 	OrkaVMMetadata string
+	OrkaVMUserdata string
 
 	OrkaEnableNodeIPMapping bool
 	OrkaNodeIPMapping       map[string]string
@@ -85,6 +87,7 @@ func ParseEnv() *Data {
 		OrkaVMUsername: getEnvWithDefault(OrkaVMUsernameEnvName, "admin"),
 		OrkaVMPassword: getEnvWithDefault(OrkaVMPasswordEnvName, "admin"),
 		OrkaVMMetadata: getEnvWithDefault(OrkaVMMetadataEnvName, ""),
+		OrkaVMUserdata: os.Getenv(OrkaVMUserdataEnvName),
 
 		OrkaEnableNodeIPMapping: getBoolEnv(OrkaEnableNodeIPMappingEnvName, false),
 
@@ -159,6 +162,17 @@ func ParseEnv() *Data {
 		}
 	}
 
+	if envData.OrkaVMUserdata == "" {
+		if userdataPath := os.Getenv(OrkaVMUserdataFilePathEnvName); userdataPath != "" {
+			userdataContent, err := os.ReadFile(userdataPath)
+			if err != nil {
+				errors = append(errors, fmt.Sprintf("failed to read the userdata script from %s: %s", OrkaVMUserdataFilePathEnvName, err))
+			} else {
+				envData.OrkaVMUserdata = string(userdataContent)
+			}
+		}
+	}
+
 	if envData.OrkaEnableNodeIPMapping {
 		err := json.Unmarshal([]byte(os.Getenv(OrkaNodeIPMappingEnvName)), &envData.OrkaNodeIPMapping)
 		if err != nil {
@@ -281,6 +295,10 @@ func validateEnv(envData *Data) []string {
 		errors = append(errors, fmt.Sprintf("%s must be formatted as key=value comma separated string", OrkaVMMetadataEnvName))
 	}
 
+	if envData.OrkaVMUserdata != "" && base64.StdEncoding.EncodedLen(len(envData.OrkaVMUserdata)) > maxUserdataEncodedSize {
+		errors = append(errors, fmt.Sprintf("the userdata script exceeds the maximum size of %d bytes when base64-encoded. Provide a smaller script via %s or %s", maxUserdataEncodedSize, OrkaVMUserdataEnvName, OrkaVMUserdataFilePathEnvName))
+	}
+
 	return errors
 }
 

pkg/env/env_test.go

@@ -1,6 +1,7 @@
 package env
 
 import (
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -31,6 +32,31 @@ var _ = Describe("Env Test", func() {
 		Entry("with invalid string with no equals sign, should be invalid", "key1;value1", false),
 	)
 
+	DescribeTable("when validating the userdata size",
+		func(scriptSize int, expectError bool) {
+			envData := &Data{
+				GitHubURL:      "https://github.com/my-org/my-repo",
+				OrkaURL:        "http://10.221.188.20",
+				OrkaToken:      "token",
+				OrkaVMConfig:   "my-config",
+				OrkaVMUserdata: strings.Repeat("a", scriptSize),
+			}
+
+			errors := validateEnv(envData)
+
+			if expectError {
+				Expect(errors).To(HaveLen(1))
+				Expect(errors[0]).To(ContainSubstring("userdata script exceeds the maximum size"))
+			} else {
+				Expect(errors).To(BeEmpty())
+			}
+		},
+		Entry("with an empty script, should be valid", 0, false),
+		Entry("with a small script, should be valid", 100, false),
+		Entry("with the largest script that encodes to 64KiB, should be valid", 49152, false),
+		Entry("with a script that encodes to over 64KiB, should be invalid", 49153, true),
+	)
+
 	DescribeTable("when detecting an enterprise config URL",
 		func(input string, expected bool) {
 			Expect(IsEnterpriseConfigURL(input)).To(Equal(expected))

pkg/orka/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -20,20 +21,31 @@ type OrkaService interface {
 
 type OrkaClient struct {
 	envData *env.Data
+	// userdataPath is a private file holding the userdata script. The file
+	// path (not the script content) is passed to orka3, which reads and
+	// base64-encodes it, so the script never appears in command lines or logs.
+	userdataPath string
 }
 
 func (client *OrkaClient) DeployVM(ctx context.Context, namePrefix, vmConfig string) (*OrkaVMDeployResponseModel, error) {
+	res, err := exec.ExecJSONCommand[[]*OrkaVMDeployResponseModel]("orka3", client.deployArgs(namePrefix, vmConfig))
+	if err != nil {
+		return nil, err
+	}
+
+	return (*res)[0], nil
+}
+
+func (client *OrkaClient) deployArgs(namePrefix, vmConfig string) []string {
 	args := []string{"vm", "deploy", namePrefix, "--config", vmConfig, "--generate-name", "-o", "json", "--namespace", client.envData.OrkaNamespace}
 	if client.envData.OrkaVMMetadata != "" {
 		args = append(args, "--metadata", client.envData.OrkaVMMetadata)
 	}
-
-	res, err := exec.ExecJSONCommand[[]*OrkaVMDeployResponseModel]("orka3", args)
-	if err != nil {
-		return nil, err
+	if client.userdataPath != "" {
+		args = append(args, "--userdata", client.userdataPath)
 	}
 
-	return (*res)[0], nil
+	return args
 }
 
 func (client *OrkaClient) ListVMs(ctx context.Context, namePrefix string) ([]*OrkaVMInfo, error) {
@@ -95,11 +107,41 @@ func NewOrkaClient(envData *env.Data, ctx context.Context) (*OrkaClient, error)
 		return nil, err
 	}
 
+	userdataPath := ""
+	if envData.OrkaVMUserdata != "" {
+		deployHelp, err := exec.ExecStringCommand("orka3", []string{"vm", "deploy", "--help"})
+		if err != nil || !strings.Contains(deployHelp, "--userdata") {
+			return nil, fmt.Errorf("the installed orka3 CLI does not support the --userdata flag. Userdata requires orka3 3.7.0 or later; upgrade the CLI or unset %s", env.OrkaVMUserdataEnvName)
+		}
+
+		if userdataPath, err = writeUserdataFile(envData.OrkaVMUserdata); err != nil {
+			return nil, fmt.Errorf("failed to prepare the userdata script: %s", err.Error())
+		}
+	}
+
 	return &OrkaClient{
-		envData: envData,
+		envData:      envData,
+		userdataPath: userdataPath,
 	}, nil
 }
 
+// writeUserdataFile stores the userdata script in a file readable only by the
+// current user, so deploys can reference it by path instead of passing the
+// script content on the orka3 command line.
+func writeUserdataFile(script string) (string, error) {
+	file, err := os.CreateTemp("", "orka-vm-userdata-*.sh")
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	if _, err := file.WriteString(script); err != nil {
+		return "", err
+	}
+
+	return file.Name(), nil
+}
+
 type OrkaTransport struct {
 	Token string
 }

pkg/orka/client_test.go

@@ -0,0 +1,71 @@
+package orka
+
+import (
+	"os"
+	"testing"
+
+	"github.com/macstadium/orka-github-actions-integration/pkg/env"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestOrka(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Orka Suite")
+}
+
+var _ = Describe("Orka Client Test", func() {
+	Describe("when building the deploy command arguments", func() {
+		It("should not include --userdata when no userdata is configured", func() {
+			client := &OrkaClient{envData: &env.Data{OrkaNamespace: "orka-default"}}
+
+			args := client.deployArgs("runner", "my-config")
+
+			Expect(args).To(Equal([]string{"vm", "deploy", "runner", "--config", "my-config",
+				"--generate-name", "-o", "json", "--namespace", "orka-default"}))
+		})
+
+		It("should include the userdata file path and not the script content", func() {
+			client := &OrkaClient{
+				envData:      &env.Data{OrkaNamespace: "orka-default", OrkaVMUserdata: "#!/bin/bash\necho hi"},
+				userdataPath: "/private/path/orka-vm-userdata-123.sh",
+			}
+
+			args := client.deployArgs("runner", "my-config")
+
+			Expect(args).To(ContainElements("--userdata", "/private/path/orka-vm-userdata-123.sh"))
+			for _, arg := range args {
+				Expect(arg).NotTo(ContainSubstring("echo hi"))
+			}
+		})
+
+		It("should keep the metadata argument alongside userdata", func() {
+			client := &OrkaClient{
+				envData:      &env.Data{OrkaNamespace: "orka-default", OrkaVMMetadata: "key=value"},
+				userdataPath: "/private/path/orka-vm-userdata-123.sh",
+			}
+
+			args := client.deployArgs("runner", "my-config")
+
+			Expect(args).To(ContainElements("--metadata", "key=value", "--userdata", "/private/path/orka-vm-userdata-123.sh"))
+		})
+	})
+
+	Describe("when writing the userdata script file", func() {
+		It("should create a file readable only by the current user with the script content", func() {
+			script := "#!/bin/bash\necho userdata"
+
+			path, err := writeUserdataFile(script)
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(os.Remove, path)
+
+			info, err := os.Stat(path)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600)))
+
+			content, err := os.ReadFile(path)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(content)).To(Equal(script))
+		})
+	})
+})