diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..3c4e9b4 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,122 @@ +--- +name: 'Integration Tests' + +# These create real instances and real disks in a real GCP project, so they are +# never run automatically on a pull request: secrets are not available to forks, +# and every run costs money. Maintainers trigger them by hand, and they run +# weekly against main so that a GCE-side change is noticed before a release. +'on': + workflow_dispatch: + inputs: + suites: + description: "Regexp of suites to run, e.g. 'default|windows'. Empty runs all of them." + required: false + default: "" + schedule: + - cron: "0 5 * * 1" + +# A project has a finite CPU quota per region. Two runs at once exhaust it and +# both fail, so let an in-flight run finish rather than cancelling it. +concurrency: + group: gce-integration + cancel-in-progress: false + +permissions: + contents: read + id-token: write + +jobs: + integration: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + GCE_PROJECT: ${{ secrets.GCE_PROJECT }} + KITCHEN_GCE_ZONE: ${{ vars.KITCHEN_GCE_ZONE || 'us-central1-a' }} + KITCHEN_GCE_REGION: ${{ vars.KITCHEN_GCE_REGION || 'us-central1' }} + KITCHEN_GCE_EMAIL: ${{ secrets.GCE_SERVICE_ACCOUNT }} + KITCHEN_SSH_KEY: ${{ github.workspace }}/.ssh/id_kitchen_gce + KITCHEN_RUN_ID: ${{ github.run_id }} + steps: + # Fail immediately, and legibly, rather than after a checkout and a + # bundle install, if the repository has not been configured for this. + - name: Check the project is configured + run: | + set -eu + if [ -z "${GCE_PROJECT}" ]; then + echo "::error::The GCE_PROJECT secret is not set - see integration/README.md" + exit 1 + fi + + - name: Checkout + uses: actions/checkout@v7 + + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + + # Workload identity federation exchanges GitHub's OIDC token for Google + # credentials, so no service account key is stored anywhere. The driver + # picks the result up as Application Default Credentials. + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + project_id: ${{ secrets.GCE_PROJECT }} + workload_identity_provider: ${{ secrets.GCE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCE_SERVICE_ACCOUNT }} + + - name: Install the gcloud CLI + uses: google-github-actions/setup-gcloud@v3 + + # The driver does not manage SSH keys. The public half travels to the + # instance as metadata; see integration/kitchen.yml. + - name: Generate an SSH key for the run + run: | + set -eu + mkdir -p "$(dirname "${KITCHEN_SSH_KEY}")" + ssh-keygen -t ed25519 -N "" -C "kitchen-google-integration" -f "${KITCHEN_SSH_KEY}" + + # Nothing in the default VPC allows WinRM, and the driver cannot open it: + # its startup script runs inside the guest. Without this the windows suite + # waits out its timeout on a running instance. + - name: Ensure the WinRM firewall rule exists + run: | + set -eu + gcloud compute firewall-rules describe kitchen-google-integration-winrm \ + --project "${GCE_PROJECT}" >/dev/null 2>&1 && exit 0 + gcloud compute firewall-rules create kitchen-google-integration-winrm \ + --project "${GCE_PROJECT}" \ + --allow tcp:5985 \ + --target-tags kitchen-google-integration \ + --source-ranges 0.0.0.0/0 + + # Via the environment rather than inline, so the input cannot be read as + # part of the command line. + - name: Run the integration suites + working-directory: integration + env: + SUITES: ${{ github.event.inputs.suites }} + run: bundle exec kitchen test "${SUITES}" --concurrency 4 + + # Destroy runs whatever happened above. A suite that leaks an instance on + # failure turns a red build into a recurring bill. + - name: Destroy everything + if: always() + working-directory: integration + run: bundle exec kitchen destroy --concurrency 4 + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: kitchen-logs + path: integration/.kitchen/logs/ + retention-days: 7 + + # Belt and braces: if destroy could not run, say so loudly rather than + # leaving instances and disks to be found on the next bill. + - name: Warn about anything left behind + if: failure() + run: | + echo "::warning::If 'Destroy everything' did not succeed, look for instances and disks labelled run-id=${KITCHEN_RUN_ID}" diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index ef4ebd8..aa769eb 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -8,6 +8,7 @@ jobs: lint-unit: uses: test-kitchen/.github/.github/workflows/lint-unit.yml@main with: - # Documentation and debugging gems are not needed to lint or test. - # YARD in particular must never gate CI. - bundle_without: "development:docs" + # Documentation gems are not needed to lint or test, and the integration + # group is only for the suites in integration/, which never run on a pull + # request. YARD in particular must never gate CI. + bundle_without: "development:docs:integration" diff --git a/.gitignore b/.gitignore index b3a42c8..8817284 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ doc/ # RSpec example status persistence spec/examples.txt + +# Test Kitchen integration run state +integration/.kitchen/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b7bc53..69becb8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,14 +50,31 @@ bundle exec cookstyle -a The unit tests mock the Google Compute Engine API, so they do not create real instances and do not require GCP credentials. -### Manual testing against GCE - -Changes that touch instance creation should also be exercised against a real -project, since the unit tests cannot catch API-level regressions. Set up -[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), -point a `kitchen.yml` at a project you control, and run `kitchen test`. -Remember that this creates billable resources — confirm the instances are gone -with `kitchen destroy` and check the GCE console afterwards. +### Integration tests + +Mocking the API proves the driver builds the right request, not that GCE accepts +it. The suites in [`integration/`](integration/README.md) close that gap: each +one creates a real instance and asserts, from inside the guest, that the driver +configured it as asked. + +```sh +export GCE_PROJECT=my-gcp-project +ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_kitchen_gce + +bundle exec rake integration:list +bundle exec rake integration:test +bundle exec rake integration:destroy # after a failed run +``` + +They are not part of `bundle exec rake` — they create billable resources — and +they never run on a pull request. Maintainers run them on demand, and weekly +against `main`. See [`integration/README.md`](integration/README.md) for what +each suite covers and how to set the project up. + +Changes that touch instance creation should be exercised this way before +merging, since the unit tests cannot catch an API-level regression. `kitchen +test` destroys on success but leaves a failed instance running, so confirm with +`rake integration:destroy` and check the GCE console afterwards. ## Submitting changes diff --git a/Gemfile b/Gemfile index 7441b52..3a51a83 100644 --- a/Gemfile +++ b/Gemfile @@ -13,3 +13,11 @@ end group :docs do gem "yard" end + +# Only needed to run the suites in integration/, which create real GCE +# instances. `bundle install --without integration` skips them. +group :integration do + gem "winrm", "~> 2.3" + gem "winrm-elevated", "~> 1.2" + gem "winrm-fs", "~> 1.3" +end diff --git a/Rakefile b/Rakefile index 8fa96bd..120310b 100644 --- a/Rakefile +++ b/Rakefile @@ -34,6 +34,25 @@ rescue LoadError puts "yard is not available. (sudo) gem install yard to generate documentation." end +namespace :integration do + # Deliberately not part of any default task: these create real instances and + # real disks in a real project, and cost real money. + desc "Run the integration suites against GCE (requires a GCP project)" + task :test do + Dir.chdir("integration") { sh "bundle exec kitchen test --concurrency 4" } + end + + desc "Destroy anything the integration suites left behind" + task :destroy do + Dir.chdir("integration") { sh "bundle exec kitchen destroy --concurrency 4" } + end + + desc "List the integration suites" + task :list do + Dir.chdir("integration") { sh "bundle exec kitchen list" } + end +end + # Documentation is intentionally NOT part of the default task: missing YARD # comments should never fail CI. task default: %i{test style} diff --git a/integration/README.md b/integration/README.md new file mode 100644 index 0000000..78e6125 --- /dev/null +++ b/integration/README.md @@ -0,0 +1,142 @@ +# Integration suites + +The unit suite stubs the Compute Engine client, so it can prove the driver +*builds* the right request but never that GCE accepts it. These suites close +that gap: each one creates a real instance and asserts, from inside the guest, +that the driver configured it the way the suite asked for. + +They are not part of `rake default`. They create real instances and real disks, +and cost real money. + +## What each suite covers + +| Suite | What it proves | +| --- | --- | +| `default` | Create, converge and destroy from an image family, with one boot disk and an external IP. The driver's own metadata reaches the instance alongside the user's. | +| `region` | With `region` and no `zone`, the driver lists the region's zones, picks one that is up, and the instance lands there. | +| `extra-disk` | A second persistent disk is created standalone, waited on until `READY`, attached, and deleted again on destroy. | +| `local-ssd` | A `local-ssd` disk is attached as `SCRATCH`, at GCE's fixed 375 GB, with no size sent for it. | +| `legacy-disk` | The deprecated top-level `disk_size` / `disk_type` / `autodelete_disk` options still produce a working boot disk. | +| `boot-disk-size` | A 10 GB request against a 20 GB image is raised to the image's size rather than rejected by GCE. | +| `metadata` | Custom metadata, network tags and `service_account_scopes` reach the instance, and short scope aliases such as `storage-ro` are expanded. | +| `preemptible` | `preemptible: true` is honoured, and auto-restart and live migration are forced off however the suite asks for them. | +| `long-instance-and-disk-names` | A suite and disk name that overflow GCE's 63-character budget fall back to a UUID instance name, leaving room for a legal disk name. | +| `windows` | The WinRM path: the guest agent resets the password for a non-builtin account over the serial port, and the driver's startup script opens 5985 inside the guest. | + +## Running them + +You need a GCP project with the Compute Engine API enabled, credentials with +rights to create instances, disks and firewall rules (see +[Authentication](../README.md#authentication)), and enough CPU quota in the +target region for four small instances at once. + +```sh +bundle install +export GCE_PROJECT=my-gcp-project +ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_kitchen_gce + +cd integration +bundle exec kitchen list +bundle exec kitchen test default-ubuntu-2204 +``` + +Or from the repository root: + +```sh +bundle exec rake integration:list +bundle exec rake integration:test # everything +bundle exec rake integration:destroy # clean up after a failed run +``` + +`kitchen test` destroys on success. It leaves the instance up on failure so you +can log in and look, so **run `kitchen destroy` when you are done** — or +`rake integration:destroy`, which does it for every suite. + +### Settings + +| Variable | Default | Purpose | +| --- | --- | --- | +| `GCE_PROJECT` | *none* | Required. Project to create instances in. | +| `KITCHEN_GCE_ZONE` | `us-central1-a` | Zone for every suite but `region`. | +| `KITCHEN_GCE_REGION` | `us-central1` | Region for the `region` suite. | +| `KITCHEN_GCE_USER` | `kitchen` | Login name, on both Linux and Windows. | +| `KITCHEN_SSH_KEY` | `~/.ssh/id_kitchen_gce` | Private key to connect with. The matching `.pub` is sent to the instance as `ssh-keys` metadata, so it must exist. | +| `KITCHEN_GCE_EMAIL` | `kitchen@example.com` | `email` for the Windows password exchange. | +| `KITCHEN_RUN_ID` | `local` | Written to every instance as a `run-id` label, so a leaked one can be traced back to the run that made it. | + +### SSH keys + +The driver does not manage SSH keys, so `kitchen.yml` puts the public half into +`ssh-keys` instance metadata itself. Two things follow: + +* The `.pub` file must exist before `kitchen create`, or rendering `kitchen.yml` + fails. +* If the project or the instance enforces **OS Login**, metadata SSH keys are + ignored and nothing will connect. Turn `enable-oslogin` off for these + instances, or run the suites in a project that does not require it. + +### The Windows suite + +Nothing in the default VPC allows WinRM, and the driver cannot open it — its +startup script runs *inside* the guest. Create the rule once per project, which +is what CI does: + +```sh +gcloud compute firewall-rules create kitchen-google-integration-winrm \ + --project "${GCE_PROJECT}" \ + --allow tcp:5985 \ + --target-tags kitchen-google-integration \ + --source-ranges 0.0.0.0/0 +``` + +Narrow `--source-ranges` to the addresses you run Test Kitchen from rather than +leaving it open to the internet. + +## Concurrency and quota + +The suites run four at a time. A fresh project is often capped at 8 to 24 CPUs +in a region, and the Windows suite uses a 2-vCPU machine type, so raising +`--concurrency` much past four tends to fail with a quota error rather than run +faster. + +## In CI + +[`.github/workflows/integration.yml`](../.github/workflows/integration.yml) runs +these weekly against `main`, and on demand through **Actions → Integration Tests +→ Run workflow**. It is never triggered by a pull request: secrets are not +available to forks, and every run costs money. + +CI authenticates with workload identity federation, exchanging GitHub's OIDC +token for Google credentials, so no service account key is stored anywhere. + +It needs three repository secrets, and a workload identity pool with a provider +scoped to this repository: + +| Secret | Value | +| --- | --- | +| `GCE_PROJECT` | Project to create instances in. | +| `GCE_WORKLOAD_IDENTITY_PROVIDER` | Full resource name of the provider, `projects//locations/global/workloadIdentityPools//providers/`. | +| `GCE_SERVICE_ACCOUNT` | Email of the service account the provider impersonates. Also used as the `email` for the Windows password exchange. | + +Two optional repository variables override the location: `KITCHEN_GCE_ZONE` and +`KITCHEN_GCE_REGION`. + +The service account needs `roles/compute.instanceAdmin.v1` to create instances +and disks, `roles/compute.securityAdmin` (or a narrower custom role) to create +the WinRM firewall rule once, and `roles/iam.serviceAccountUser` so it can +attach itself to the instances the `metadata` suite gives scopes to. + +## Adding a suite + +Add it to `kitchen.yml` with a script in `scripts/`. Assertions live in the +**provisioner**, not a verifier: the script is transferred over the driver's own +transport and executed on the instance, so reaching the machine at all is part +of every assertion, a non-zero exit fails the suite, and there is no verifier +licence to satisfy. + +Anything the script needs to know has to travel as instance metadata — see the +`region` suite. The provisioner runs on the instance, where nothing of the local +environment survives. + +Keep each suite pointed at one behaviour: when it fails, its name should say +what broke. diff --git a/integration/kitchen.yml b/integration/kitchen.yml new file mode 100644 index 0000000..030f425 --- /dev/null +++ b/integration/kitchen.yml @@ -0,0 +1,193 @@ +--- +# Integration suites for kitchen-google. Each one creates a real GCE instance +# and asserts, from inside the guest, that the driver configured it the way the +# suite asked for. +# +# See integration/README.md for how to run these. + +driver: + name: gce + project: <%= ENV["GCE_PROJECT"] %> + zone: <%= ENV.fetch("KITCHEN_GCE_ZONE", "us-central1-a") %> + machine_type: e2-small + labels: + created-by: kitchen-google-integration + run-id: <%= ENV.fetch("KITCHEN_RUN_ID", "local") %> + tags: + - kitchen-google-integration + # kitchen-google does not manage SSH keys, so the public half of the key the + # transport connects with has to reach the instance as metadata. This is also + # a live check that user metadata survives the merge with the driver's own. + metadata: + ssh-keys: <%= "#{ENV.fetch("KITCHEN_GCE_USER", "kitchen")}:#{File.read(File.expand_path("#{ENV.fetch("KITCHEN_SSH_KEY", "~/.ssh/id_kitchen_gce")}.pub")).strip}" %> + +# Assertions live in the provisioner rather than a verifier: the script is +# transferred over the driver's own transport and run on the instance, so +# reaching the machine at all is part of every assertion, and a non-zero exit +# fails the suite. It also keeps the suites free of a verifier licence. +provisioner: + name: shell + +verifier: + name: shell + command: echo "assertions run in the provisioner" + +transport: + name: ssh + username: <%= ENV.fetch("KITCHEN_GCE_USER", "kitchen") %> + ssh_key: <%= ENV.fetch("KITCHEN_SSH_KEY", "~/.ssh/id_kitchen_gce") %> + # An instance that can never be reached should fail in a couple of minutes, + # not after the ten-minute default. + max_wait_until_ready: 300 + +platforms: + - name: ubuntu-2204 + driver: + image_family: ubuntu-2204-lts + image_project: ubuntu-os-cloud + + # Rocky's image is 20 GB, which is larger than the driver's own 10 GB default. + # That is the whole point of the boot-disk-size suite. + - name: rockylinux-9 + driver: + image_family: rocky-linux-9 + image_project: rocky-linux-cloud + + - name: windows-2022 + driver: + image_family: windows-2022 + image_project: windows-cloud + machine_type: e2-standard-2 + email: <%= ENV.fetch("KITCHEN_GCE_EMAIL", "kitchen@example.com") %> + transport: + name: winrm + # Never "administrator": Google's Windows images ship that account + # disabled and the guest agent will not enable it. + username: kitchen + elevated: true + max_wait_until_ready: 600 + +suites: + # The ordinary path: image family, a single boot disk, an external IP. + - name: default + includes: [ubuntu-2204] + provisioner: + script: scripts/baseline.sh + + # No zone at all, so the driver has to list the region's zones and pick a + # live one. The script asserts the instance really did land in that region. + - name: region + includes: [ubuntu-2204] + driver: + zone: ~ + region: <%= ENV.fetch("KITCHEN_GCE_REGION", "us-central1") %> + metadata: + kitchen-expected-region: <%= ENV.fetch("KITCHEN_GCE_REGION", "us-central1") %> + provisioner: + script: scripts/region.sh + + # An extra persistent disk, which the driver creates standalone and attaches, + # rather than inline with the instance. + - name: extra-disk + includes: [ubuntu-2204] + driver: + disks: + boot: + boot: true + disk_size: 15 + data: + disk_size: 25 + disk_type: pd-balanced + provisioner: + script: scripts/extra_disk.sh + + # Local SSD scratch space, which is attached as a SCRATCH disk with no size + # and no source image. E2 machine types cannot carry one. + - name: local-ssd + includes: [ubuntu-2204] + driver: + machine_type: n1-standard-1 + disks: + boot: + boot: true + scratch: + disk_type: local-ssd + provisioner: + script: scripts/local_ssd.sh + + # The deprecated top-level disk options, which are still supported and still + # have to produce a working boot disk. + - name: legacy-disk + includes: [ubuntu-2204] + driver: + disk_size: 20 + disk_type: pd-balanced + autodelete_disk: true + provisioner: + script: scripts/boot_disk_size.sh + + # A 10 GB request against a 20 GB image. GCE refuses to clone an image into a + # disk smaller than itself, so the driver has to raise the request instead of + # failing the run. + - name: boot-disk-size + includes: [rockylinux-9] + driver: + disks: + boot: + boot: true + disk_size: 10 + provisioner: + script: scripts/boot_disk_size.sh + + # Metadata, labels, tags and service account scopes, all read back from the + # instance's own metadata server. + - name: metadata + includes: [ubuntu-2204] + driver: + metadata: + kitchen-integration: metadata-suite + labels: + suite: metadata + tags: + - kitchen-google-integration + - kitchen-metadata-suite + service_account_scopes: + - storage-ro + - logging-write + provisioner: + script: scripts/metadata.sh + + # Preemptible instances, which GCE will neither live-migrate nor auto-restart + # whatever the driver is asked for. + - name: preemptible + includes: [ubuntu-2204] + driver: + preemptible: true + auto_restart: true + auto_migrate: true + provisioner: + script: scripts/preemptible.sh + + # A suite name long enough that "tk---" will not fit + # once the longest disk name is reserved out of the 63-character budget, so + # the driver has to fall back to a UUID name. A disk name that overflows is + # the failure this covers: it produced a legal instance name and an illegal + # disk name. + - name: long-instance-and-disk-names + includes: [ubuntu-2204] + driver: + disks: + boot: + boot: true + data-disk-long-enough: + disk_size: 10 + provisioner: + script: scripts/long_names.sh + + # The Windows path: the guest agent resets the password for a non-builtin + # account over the serial port, and the driver's startup script opens WinRM + # inside the guest. + - name: windows + includes: [windows-2022] + provisioner: + script: scripts/windows.ps1 diff --git a/integration/scripts/baseline.sh b/integration/scripts/baseline.sh new file mode 100755 index 0000000..c2f79cb --- /dev/null +++ b/integration/scripts/baseline.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# The path every suite depends on: the instance exists, the driver's own +# metadata reached it, and it is reachable over the transport the driver +# handed to Test Kitchen. Running at all proves the last of those. +set -eu + +MD="http://metadata.google.internal/computeMetadata/v1" +md() { curl -sf -H "Metadata-Flavor: Google" "${MD}/$1"; } + +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo "OK: $*"; } + +name=$(md instance/name) +zone=$(md instance/zone | awk -F/ '{print $NF}') +echo "instance=${name} zone=${zone}" + +[ -n "${name}" ] || fail "the metadata server reported no instance name" + +# Set by the driver on every instance, whatever the suite asks for. +[ "$(md instance/attributes/created-by)" = "test-kitchen" ] || + fail "created-by metadata is not test-kitchen" + +md instance/attributes/test-kitchen-instance >/dev/null || + fail "test-kitchen-instance metadata is missing" +md instance/attributes/test-kitchen-user >/dev/null || + fail "test-kitchen-user metadata is missing" + +# The user metadata carrying the SSH key has to survive the merge with the +# driver's own keys -- GCE rejects an instance that is sent a key twice. +md instance/attributes/ssh-keys >/dev/null || + fail "the ssh-keys metadata this suite set did not reach the instance" + +ok "baseline" diff --git a/integration/scripts/boot_disk_size.sh b/integration/scripts/boot_disk_size.sh new file mode 100755 index 0000000..9bf371a --- /dev/null +++ b/integration/scripts/boot_disk_size.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Two suites share this script: +# +# legacy-disk asks for a 20 GB boot disk through the deprecated top-level +# disk_size option. +# boot-disk-size asks for 10 GB from a 20 GB image, which GCE would refuse, +# so the driver has to raise the request to the image's size. +# +# Either way the boot disk has to come out at 20 GB or more. +set -eu + +root_dev=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)") +boot_gb=$(( $(lsblk -bdn -o SIZE "/dev/${root_dev}") / 1000 / 1000 / 1000 )) +echo "boot device ${root_dev} is ${boot_gb} GB" + +[ "${boot_gb}" -ge 20 ] || + { echo "FAIL: boot disk is ${boot_gb} GB, expected at least 20" >&2; exit 1; } + +echo "OK: boot-disk-size ${boot_gb}GB" diff --git a/integration/scripts/extra_disk.sh b/integration/scripts/extra_disk.sh new file mode 100755 index 0000000..711af2b --- /dev/null +++ b/integration/scripts/extra_disk.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# A standalone persistent disk, created before the instance and attached to it. +# The driver takes a completely different path for these than for a disk +# created inline with the instance, including waiting for READY and recording +# the name so a later destroy can find it. +set -eu + +fail() { echo "FAIL: $*" >&2; exit 1; } + +root_dev=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)") +echo "root device: ${root_dev}" + +# Sizes in whole gigabytes, keyed by device name. +lsblk -bdn -o NAME,SIZE | while read -r dev bytes; do + echo " ${dev} $((bytes / 1000 / 1000 / 1000)) GB" +done + +boot_gb=$(( $(lsblk -bdn -o SIZE "/dev/${root_dev}") / 1000 / 1000 / 1000 )) +[ "${boot_gb}" -ge 15 ] || fail "boot disk is ${boot_gb} GB, expected 15" + +data_gb=$(lsblk -bdn -o NAME,SIZE | + awk -v root="${root_dev}" '$1 != root { printf "%d\n", $2 / 1000 / 1000 / 1000 }' | + sort -rn | head -1) + +[ -n "${data_gb}" ] || fail "no disk was attached other than the boot disk" +[ "${data_gb}" -ge 25 ] || fail "extra disk is ${data_gb} GB, expected 25" + +echo "OK: extra-disk boot=${boot_gb}GB data=${data_gb}GB" diff --git a/integration/scripts/local_ssd.sh b/integration/scripts/local_ssd.sh new file mode 100755 index 0000000..ca0dcf6 --- /dev/null +++ b/integration/scripts/local_ssd.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Local SSDs are attached as SCRATCH disks with no size and no source image, +# which is the one disk shape the driver must not send a disk_size for. GCE +# fixes them at 375 GB. +set -eu + +root_dev=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)") +echo "root device: ${root_dev}" +lsblk -bdn -o NAME,SIZE + +scratch=$(lsblk -bdn -o NAME,SIZE | + awk -v root="${root_dev}" '$1 != root { gb = $2 / 1000 / 1000 / 1000; if (gb >= 370 && gb <= 380) print $1 }' | + head -1) + +[ -n "${scratch}" ] || + { echo "FAIL: no 375 GB scratch disk is attached" >&2; exit 1; } + +echo "OK: local-ssd ${scratch}" diff --git a/integration/scripts/long_names.sh b/integration/scripts/long_names.sh new file mode 100755 index 0000000..6537c88 --- /dev/null +++ b/integration/scripts/long_names.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# The suite and disk names together overflow the 63 characters GCE allows, so +# the driver must fall back to a UUID-based instance name -- and must still +# leave room for "-data-disk-long-enough" to be a legal disk name. +set -eu + +MD="http://metadata.google.internal/computeMetadata/v1" +md() { curl -sf -H "Metadata-Flavor: Google" "${MD}/$1"; } + +fail() { echo "FAIL: $*" >&2; exit 1; } + +name=$(md instance/name) +echo "instance=${name} (${#name} characters)" + +[ "${#name}" -le 63 ] || fail "instance name is ${#name} characters, GCE allows 63" +echo "${name}" | grep -Eq '^[a-z]([-a-z0-9]*[a-z0-9])?$' || + fail "instance name '${name}' is not a legal GCE resource name" + +# The suite name cannot fit, so the driver should have used tk-. +echo "${name}" | grep -Eq '^tk-[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$' || + fail "expected a tk- fallback name, got '${name}'" + +# ...and the disk named after it has to have been created and attached. +root_dev=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)") +lsblk -bdn -o NAME,SIZE | awk -v root="${root_dev}" '$1 != root' | grep -q . || + fail "the extra disk was not attached" + +echo "OK: long-instance-and-disk-names" diff --git a/integration/scripts/metadata.sh b/integration/scripts/metadata.sh new file mode 100755 index 0000000..667aa29 --- /dev/null +++ b/integration/scripts/metadata.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Metadata, network tags and service account scopes, read back from the +# instance's own metadata server. Labels are not exposed there, so they are +# asserted from the API side rather than here. +set -eu + +MD="http://metadata.google.internal/computeMetadata/v1" +md() { curl -sf -H "Metadata-Flavor: Google" "${MD}/$1"; } + +fail() { echo "FAIL: $*" >&2; exit 1; } + +# The suite's own metadata key. +value=$(md instance/attributes/kitchen-integration) +echo "kitchen-integration=${value}" +[ "${value}" = "metadata-suite" ] || + fail "kitchen-integration metadata is '${value}', expected 'metadata-suite'" + +# ...alongside, not instead of, the driver's own. +[ "$(md instance/attributes/created-by)" = "test-kitchen" ] || + fail "the driver's created-by metadata was lost" + +tags=$(md instance/tags) +echo "tags=${tags}" +echo "${tags}" | grep -q "kitchen-metadata-suite" || + fail "the kitchen-metadata-suite network tag is missing" + +# The driver expands short aliases such as storage-ro into full scope URLs. +scopes=$(md instance/service-accounts/default/scopes) +echo "scopes=${scopes}" +echo "${scopes}" | grep -q "https://www.googleapis.com/auth/devstorage.read_only" || + fail "the storage-ro alias did not expand to devstorage.read_only" +echo "${scopes}" | grep -q "https://www.googleapis.com/auth/logging.write" || + fail "the logging-write alias did not expand to logging.write" + +echo "OK: metadata" diff --git a/integration/scripts/preemptible.sh b/integration/scripts/preemptible.sh new file mode 100755 index 0000000..df96a9f --- /dev/null +++ b/integration/scripts/preemptible.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# A preemptible instance can neither live-migrate nor auto-restart. The suite +# asks for both anyway, so this also covers the driver overriding them. +set -eu + +MD="http://metadata.google.internal/computeMetadata/v1" +md() { curl -sf -H "Metadata-Flavor: Google" "${MD}/$1"; } + +fail() { echo "FAIL: $*" >&2; exit 1; } + +preemptible=$(md instance/scheduling/preemptible) +restart=$(md instance/scheduling/automatic-restart) +maintenance=$(md instance/scheduling/on-host-maintenance) +echo "preemptible=${preemptible} automatic-restart=${restart} on-host-maintenance=${maintenance}" + +[ "${preemptible}" = "TRUE" ] || fail "the instance is not preemptible" +[ "${restart}" = "FALSE" ] || fail "auto_restart was not forced off" +[ "${maintenance}" = "TERMINATE" ] || fail "on-host-maintenance is not TERMINATE" + +echo "OK: preemptible" diff --git a/integration/scripts/region.sh b/integration/scripts/region.sh new file mode 100755 index 0000000..35d06cb --- /dev/null +++ b/integration/scripts/region.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# The suite configures a region and no zone, so the driver had to list the +# region's zones and pick one that was up. Assert it landed in that region. +# +# The expectation travels as instance metadata rather than as an environment +# variable: the provisioner runs the script on the instance, where nothing of +# the local environment survives. +set -eu + +MD="http://metadata.google.internal/computeMetadata/v1" +md() { curl -sf -H "Metadata-Flavor: Google" "${MD}/$1"; } + +expected_region=$(md instance/attributes/kitchen-expected-region) +zone=$(md instance/zone | awk -F/ '{print $NF}') +echo "zone=${zone} expected_region=${expected_region}" + +[ -n "${expected_region}" ] || + { echo "FAIL: kitchen-expected-region metadata is missing" >&2; exit 1; } + +# us-central1-a -> us-central1 +region=$(echo "${zone}" | sed 's/-[a-z]$//') + +[ "${region}" = "${expected_region}" ] || + { echo "FAIL: instance is in region ${region}, not ${expected_region}" >&2; exit 1; } + +echo "OK: region" diff --git a/integration/scripts/windows.ps1 b/integration/scripts/windows.ps1 new file mode 100644 index 0000000..96ec9ea --- /dev/null +++ b/integration/scripts/windows.ps1 @@ -0,0 +1,48 @@ +# The Windows path end to end. Reaching this script at all is most of the +# assertion: it arrived over WinRM, as the account the guest agent created and +# whose password the driver decrypted off the serial port. +$ErrorActionPreference = "Stop" + +function Get-GceMetadata($Path) { + Invoke-RestMethod -Headers @{ "Metadata-Flavor" = "Google" } ` + -Uri "http://metadata.google.internal/computeMetadata/v1/$Path" +} + +function Fail($Message) { + Write-Error "FAIL: $Message" + exit 1 +} + +$name = Get-GceMetadata "instance/name" +Write-Host "instance=$name user=$env:USERNAME" + +if ((Get-GceMetadata "instance/attributes/created-by") -ne "test-kitchen") { + Fail "created-by metadata is not test-kitchen" +} + +# The driver adds this for WinRM transports only. It is what opens 5985 inside +# the guest -- the VPC firewall rule has to be created separately. +$startup = Get-GceMetadata "instance/attributes/windows-startup-script-ps1" +if ($startup -notmatch "localport=5985") { + Fail "the WinRM startup script metadata was not set" +} + +# Google's images ship the built-in Administrator disabled and the agent will +# not enable it, so the driver must be connecting as something else. +if ($env:USERNAME -ieq "administrator") { + Fail "connected as the built-in Administrator, which should never work" +} + +$admins = Get-LocalGroupMember -Group "Administrators" | ForEach-Object { $_.Name } +Write-Host "Administrators: $($admins -join ', ')" +if (-not ($admins -match [regex]::Escape($env:USERNAME))) { + Fail "$env:USERNAME is not in the local Administrators group" +} + +# The password the driver decrypted is what authenticated this session. +$listeners = winrm enumerate winrm/config/listener +if ($listeners -notmatch "5985") { + Fail "no WinRM listener on 5985" +} + +Write-Host "OK: windows"