From f207fd133b8385f57275f57241e023b92495967a Mon Sep 17 00:00:00 2001 From: Robert Elwell Date: Tue, 1 Sep 2026 14:22:20 -0400 Subject: [PATCH] Add userdata support for runner VMs Adds optional ORKA_VM_USERDATA / ORKA_VM_USERDATA_FILE_PATH settings: a bootstrap script executed as root inside each runner VM on its first boot, before the GitHub runner starts. This prepares the integration for Orka 3.7, which introduces VM userdata - one base image can serve many runner pools without baking bootstrap steps into the image. - The script (from env content or a file, mirroring the GitHub App private key pattern) is validated against the 64KiB encoded size cap at startup and stored in a file readable only by the current user; deploys pass the file path to orka3 --userdata, so the script content never appears in command lines or error logs - Startup fails fast with a clear error when the installed orka3 CLI predates the --userdata flag (requires orka3 3.7.0+) - Userdata never delays or gates runner VM boot; output is written to /var/log/orka/userdata.log inside the VM Co-Authored-By: Claude Fable 5 --- README.md | 2 ++ examples/.env | 13 ++++++++ pkg/env/constants.go | 16 +++++++--- pkg/env/env.go | 18 +++++++++++ pkg/env/env_test.go | 26 +++++++++++++++ pkg/orka/client.go | 54 +++++++++++++++++++++++++++---- pkg/orka/client_test.go | 71 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 pkg/orka/client_test.go diff --git a/README.md b/README.md index 67d885b..e28fb3c 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ The Orka GitHub runner requires the following environment variabales to be confi * `ORKA_VM_USERNAME`: Specifies the username for the deployed VMs. If no value is provided, it defaults to admin. * `ORKA_VM_PASSWORD`: Specifies the password for the deployed VMs. If no value is provided, it defaults to admin. * `ORKA_VM_METADATA`: Specifies custom VM metadata passed to the VM. Must be formatted as key=value comma separated pairs. +* `ORKA_VM_USERDATA`: Specifies a userdata bootstrap script executed as root inside each runner VM on its first boot, before the GitHub runner starts. Use it to prepare the VM (install tools, enroll in MDM, register services) without baking the steps into the VM image. The script runs in the background and never delays or fails the VM boot; its output is written to `/var/log/orka/userdata.log` inside the VM. The script content is kept out of the integration's logs. Requires Orka 3.7.0 or later (both the cluster and the `orka3` CLI) and an Apple Silicon VM. +* `ORKA_VM_USERDATA_FILE_PATH`: Specifies a path to a file containing the userdata bootstrap script. Ignored when `ORKA_VM_USERDATA` is set. * `ORKA_ENABLE_NODE_IP_MAPPING`: Specifies whether to enable the mapping of Orka node IPs to external IPs. * `ORKA_NODE_IP_MAPPING`: Defines the mapping of Orka node internal IPs to external host IPs. * `RUNNERS`: A JSON array containing configuration details of the GitHub runner scale set that will be created. Currently only one runner is supported. See [here](#how-to-use-multiple-runners) for how to use multiple runners. Example usage: `RUNNERS='[{"name":"my-github-runner", "id": 1}]'`. The `name` field should match the value specified in the `runs-on` field in the Actions workflow. The `id` field should be used to differentiate runners with GitHub. We default to `1` if it is not defined. See an example [here](./examples/ci.yml). diff --git a/examples/.env b/examples/.env index 6749168..77a9d43 100644 --- a/examples/.env +++ b/examples/.env @@ -98,6 +98,19 @@ LOG_LEVEL="debug" # Should be formatted as "key=value" comma separated list. ORKA_VM_METADATA="key1=value1,key2=value2" +# [Optional] ORKA_VM_USERDATA specifies a bootstrap script executed as root inside each +# runner VM on its first boot, before the GitHub runner starts. Use it to prepare the VM +# (install tools, enroll in MDM, register services) without baking the steps into the image. +# The script runs in the background, never delays or fails the VM boot, and its output is +# written to /var/log/orka/userdata.log inside the VM. +# Requires Orka 3.7.0 or later (cluster and orka3 CLI) and an Apple Silicon VM. +# ORKA_VM_USERDATA="#!/bin/bash +# echo hello > /Users/Shared/bootstrap-done.txt" + +# [Optional] ORKA_VM_USERDATA_FILE_PATH specifies a path to a file containing the userdata +# bootstrap script. Ignored when ORKA_VM_USERDATA is set. +# ORKA_VM_USERDATA_FILE_PATH="/path/to/bootstrap.sh" + # [Optional] VM_TRACKER_INTERVAL specifies the interval at which the VM tracker will check for orphaned VMs. # VMs are deleted if they do not have a corresponding GitHub runner for 2 consecutive checks. # If not provided, it defaults to 300 seconds. diff --git a/pkg/env/constants.go b/pkg/env/constants.go index 26c65a5..7db548a 100644 --- a/pkg/env/constants.go +++ b/pkg/env/constants.go @@ -1,5 +1,9 @@ package env +// maxUserdataEncodedSize is the maximum size of the base64-encoded userdata +// script accepted by the Orka API (64KiB). +const maxUserdataEncodedSize = 65536 + const ( GitHubAppIDEnvName = "GITHUB_APP_ID" GitHubAppInstallationIDEnvName = "GITHUB_APP_INSTALLATION_ID" @@ -14,11 +18,13 @@ const ( OrkaURLEnvName = "ORKA_URL" OrkaTokenEnvName = "ORKA_TOKEN" - OrkaNamespaceEnvName = "ORKA_NAMESPACE" - OrkaVMConfigEnvName = "ORKA_VM_CONFIG" - OrkaVMUsernameEnvName = "ORKA_VM_USERNAME" - OrkaVMPasswordEnvName = "ORKA_VM_PASSWORD" - OrkaVMMetadataEnvName = "ORKA_VM_METADATA" + OrkaNamespaceEnvName = "ORKA_NAMESPACE" + OrkaVMConfigEnvName = "ORKA_VM_CONFIG" + OrkaVMUsernameEnvName = "ORKA_VM_USERNAME" + OrkaVMPasswordEnvName = "ORKA_VM_PASSWORD" + OrkaVMMetadataEnvName = "ORKA_VM_METADATA" + OrkaVMUserdataEnvName = "ORKA_VM_USERDATA" + OrkaVMUserdataFilePathEnvName = "ORKA_VM_USERDATA_FILE_PATH" OrkaEnableNodeIPMappingEnvName = "ORKA_ENABLE_NODE_IP_MAPPING" OrkaNodeIPMappingEnvName = "ORKA_NODE_IP_MAPPING" diff --git a/pkg/env/env.go b/pkg/env/env.go index 3f7b643..33ec999 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -1,6 +1,7 @@ package env import ( + "encoding/base64" "encoding/json" "fmt" "log" @@ -40,6 +41,7 @@ type Data struct { OrkaVMUsername string OrkaVMPassword string OrkaVMMetadata string + OrkaVMUserdata string OrkaEnableNodeIPMapping bool OrkaNodeIPMapping map[string]string @@ -85,6 +87,7 @@ func ParseEnv() *Data { OrkaVMUsername: getEnvWithDefault(OrkaVMUsernameEnvName, "admin"), OrkaVMPassword: getEnvWithDefault(OrkaVMPasswordEnvName, "admin"), OrkaVMMetadata: getEnvWithDefault(OrkaVMMetadataEnvName, ""), + OrkaVMUserdata: os.Getenv(OrkaVMUserdataEnvName), OrkaEnableNodeIPMapping: getBoolEnv(OrkaEnableNodeIPMappingEnvName, false), @@ -159,6 +162,17 @@ func ParseEnv() *Data { } } + if envData.OrkaVMUserdata == "" { + if userdataPath := os.Getenv(OrkaVMUserdataFilePathEnvName); userdataPath != "" { + userdataContent, err := os.ReadFile(userdataPath) + if err != nil { + errors = append(errors, fmt.Sprintf("failed to read the userdata script from %s: %s", OrkaVMUserdataFilePathEnvName, err)) + } else { + envData.OrkaVMUserdata = string(userdataContent) + } + } + } + if envData.OrkaEnableNodeIPMapping { err := json.Unmarshal([]byte(os.Getenv(OrkaNodeIPMappingEnvName)), &envData.OrkaNodeIPMapping) if err != nil { @@ -281,6 +295,10 @@ func validateEnv(envData *Data) []string { errors = append(errors, fmt.Sprintf("%s must be formatted as key=value comma separated string", OrkaVMMetadataEnvName)) } + if envData.OrkaVMUserdata != "" && base64.StdEncoding.EncodedLen(len(envData.OrkaVMUserdata)) > maxUserdataEncodedSize { + errors = append(errors, fmt.Sprintf("the userdata script exceeds the maximum size of %d bytes when base64-encoded. Provide a smaller script via %s or %s", maxUserdataEncodedSize, OrkaVMUserdataEnvName, OrkaVMUserdataFilePathEnvName)) + } + return errors } diff --git a/pkg/env/env_test.go b/pkg/env/env_test.go index 79515c3..5a857f9 100644 --- a/pkg/env/env_test.go +++ b/pkg/env/env_test.go @@ -1,6 +1,7 @@ package env import ( + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -31,6 +32,31 @@ var _ = Describe("Env Test", func() { Entry("with invalid string with no equals sign, should be invalid", "key1;value1", false), ) + DescribeTable("when validating the userdata size", + func(scriptSize int, expectError bool) { + envData := &Data{ + GitHubURL: "https://github.com/my-org/my-repo", + OrkaURL: "http://10.221.188.20", + OrkaToken: "token", + OrkaVMConfig: "my-config", + OrkaVMUserdata: strings.Repeat("a", scriptSize), + } + + errors := validateEnv(envData) + + if expectError { + Expect(errors).To(HaveLen(1)) + Expect(errors[0]).To(ContainSubstring("userdata script exceeds the maximum size")) + } else { + Expect(errors).To(BeEmpty()) + } + }, + Entry("with an empty script, should be valid", 0, false), + Entry("with a small script, should be valid", 100, false), + Entry("with the largest script that encodes to 64KiB, should be valid", 49152, false), + Entry("with a script that encodes to over 64KiB, should be invalid", 49153, true), + ) + DescribeTable("when detecting an enterprise config URL", func(input string, expected bool) { Expect(IsEnterpriseConfigURL(input)).To(Equal(expected)) diff --git a/pkg/orka/client.go b/pkg/orka/client.go index 0a9b764..1fc3264 100644 --- a/pkg/orka/client.go +++ b/pkg/orka/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "os" "strings" "time" @@ -20,20 +21,31 @@ type OrkaService interface { type OrkaClient struct { envData *env.Data + // userdataPath is a private file holding the userdata script. The file + // path (not the script content) is passed to orka3, which reads and + // base64-encodes it, so the script never appears in command lines or logs. + userdataPath string } func (client *OrkaClient) DeployVM(ctx context.Context, namePrefix, vmConfig string) (*OrkaVMDeployResponseModel, error) { + res, err := exec.ExecJSONCommand[[]*OrkaVMDeployResponseModel]("orka3", client.deployArgs(namePrefix, vmConfig)) + if err != nil { + return nil, err + } + + return (*res)[0], nil +} + +func (client *OrkaClient) deployArgs(namePrefix, vmConfig string) []string { args := []string{"vm", "deploy", namePrefix, "--config", vmConfig, "--generate-name", "-o", "json", "--namespace", client.envData.OrkaNamespace} if client.envData.OrkaVMMetadata != "" { args = append(args, "--metadata", client.envData.OrkaVMMetadata) } - - res, err := exec.ExecJSONCommand[[]*OrkaVMDeployResponseModel]("orka3", args) - if err != nil { - return nil, err + if client.userdataPath != "" { + args = append(args, "--userdata", client.userdataPath) } - return (*res)[0], nil + return args } func (client *OrkaClient) ListVMs(ctx context.Context, namePrefix string) ([]*OrkaVMInfo, error) { @@ -95,11 +107,41 @@ func NewOrkaClient(envData *env.Data, ctx context.Context) (*OrkaClient, error) return nil, err } + userdataPath := "" + if envData.OrkaVMUserdata != "" { + deployHelp, err := exec.ExecStringCommand("orka3", []string{"vm", "deploy", "--help"}) + if err != nil || !strings.Contains(deployHelp, "--userdata") { + return nil, fmt.Errorf("the installed orka3 CLI does not support the --userdata flag. Userdata requires orka3 3.7.0 or later; upgrade the CLI or unset %s", env.OrkaVMUserdataEnvName) + } + + if userdataPath, err = writeUserdataFile(envData.OrkaVMUserdata); err != nil { + return nil, fmt.Errorf("failed to prepare the userdata script: %s", err.Error()) + } + } + return &OrkaClient{ - envData: envData, + envData: envData, + userdataPath: userdataPath, }, nil } +// writeUserdataFile stores the userdata script in a file readable only by the +// current user, so deploys can reference it by path instead of passing the +// script content on the orka3 command line. +func writeUserdataFile(script string) (string, error) { + file, err := os.CreateTemp("", "orka-vm-userdata-*.sh") + if err != nil { + return "", err + } + defer file.Close() + + if _, err := file.WriteString(script); err != nil { + return "", err + } + + return file.Name(), nil +} + type OrkaTransport struct { Token string } diff --git a/pkg/orka/client_test.go b/pkg/orka/client_test.go new file mode 100644 index 0000000..908c687 --- /dev/null +++ b/pkg/orka/client_test.go @@ -0,0 +1,71 @@ +package orka + +import ( + "os" + "testing" + + "github.com/macstadium/orka-github-actions-integration/pkg/env" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOrka(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Orka Suite") +} + +var _ = Describe("Orka Client Test", func() { + Describe("when building the deploy command arguments", func() { + It("should not include --userdata when no userdata is configured", func() { + client := &OrkaClient{envData: &env.Data{OrkaNamespace: "orka-default"}} + + args := client.deployArgs("runner", "my-config") + + Expect(args).To(Equal([]string{"vm", "deploy", "runner", "--config", "my-config", + "--generate-name", "-o", "json", "--namespace", "orka-default"})) + }) + + It("should include the userdata file path and not the script content", func() { + client := &OrkaClient{ + envData: &env.Data{OrkaNamespace: "orka-default", OrkaVMUserdata: "#!/bin/bash\necho hi"}, + userdataPath: "/private/path/orka-vm-userdata-123.sh", + } + + args := client.deployArgs("runner", "my-config") + + Expect(args).To(ContainElements("--userdata", "/private/path/orka-vm-userdata-123.sh")) + for _, arg := range args { + Expect(arg).NotTo(ContainSubstring("echo hi")) + } + }) + + It("should keep the metadata argument alongside userdata", func() { + client := &OrkaClient{ + envData: &env.Data{OrkaNamespace: "orka-default", OrkaVMMetadata: "key=value"}, + userdataPath: "/private/path/orka-vm-userdata-123.sh", + } + + args := client.deployArgs("runner", "my-config") + + Expect(args).To(ContainElements("--metadata", "key=value", "--userdata", "/private/path/orka-vm-userdata-123.sh")) + }) + }) + + Describe("when writing the userdata script file", func() { + It("should create a file readable only by the current user with the script content", func() { + script := "#!/bin/bash\necho userdata" + + path, err := writeUserdataFile(script) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.Remove, path) + + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + + content, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal(script)) + }) + }) +})