From 5d61bbb509b7dfb46e25ae3d807cca60c9f13d27 Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Wed, 9 Sep 2026 17:28:19 +0530 Subject: [PATCH 1/2] feat(config)!: remove legacy test-suites support --- docs/user/reference/cli/azldev_image_test.md | 24 +- docs/user/reference/config/config-file.md | 1 - docs/user/reference/config/images.md | 12 +- docs/user/reference/config/test-suites.md | 171 ------------ docs/user/reference/config/tests.md | 3 +- .../azldev/agentskill/content/image.md.tmpl | 4 +- internal/app/azldev/cmds/image/lisarunner.go | 39 +-- internal/app/azldev/cmds/image/list_test.go | 8 +- internal/app/azldev/cmds/image/test.go | 145 ++-------- .../azldev/cmds/image/test_internal_test.go | 52 +--- internal/projectconfig/configfile.go | 26 -- internal/projectconfig/configfile_test.go | 29 -- internal/projectconfig/image.go | 29 +- internal/projectconfig/loader.go | 23 -- internal/projectconfig/loader_test.go | 248 ------------------ internal/projectconfig/project.go | 36 --- internal/projectconfig/testsuite.go | 52 ---- internal/projectconfig/testsuite_test.go | 105 +------- ...ainer_config_generate-schema_stdout_1.snap | 167 ------------ ...shots_config_generate-schema_stdout_1.snap | 167 ------------ schemas/azldev.schema.json | 167 ------------ 21 files changed, 65 insertions(+), 1443 deletions(-) delete mode 100644 docs/user/reference/config/test-suites.md diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 2816ac3c5..a96075682 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -9,14 +9,11 @@ Run tests against an Azure Linux image Run tests against an Azure Linux image using test definitions declared in the project configuration. -Images may reference tests directly via [images.NAME.tests.tests] entries, or via -named [test-groups]. Legacy [test-suites] references are still supported. +Images reference tests via [images.NAME.tests.tests] entries, which resolve to +project-level [tests.X] definitions or named [test-groups]. By default, all tests associated with the named image are run. Use --test-suite to select specific test names or test-group names (may be repeated). -For images still configured with legacy [test-suites] (via 'tests.test-suites'), -the values passed to --test-suite are instead matched against those legacy -test suite names. The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -29,14 +26,13 @@ test-paths are expanded automatically. For LISA tests, the test runner executes on the host and boots the image in a QEMU VM. azldev clones the LISA framework, generates a runbook from the test's -configured criteria (or, for legacy [test-suites], its test cases), and runs it -against the image. azldev generates an ephemeral SSH key pair to access the -booted VM and removes it once the test finishes. New-style [tests.X] LISA -definitions require a [tests.X.lisa.source] (git-url, ref) to run locally; -without one, the test is metadata-only and must be run through the LISA -infrastructure. Use --lisa-dir to run against an already-cloned LISA checkout -instead of cloning; this also allows running new-style LISA tests that have no -[tests.X.lisa.source] configured. +configured criteria, and runs it against the image. azldev generates an +ephemeral SSH key pair to access the booted VM and removes it once the test +finishes. [tests.X] LISA definitions require a [tests.X.lisa.source] (git-url, +ref) to run locally; without one, the test is metadata-only and must be run +through the LISA infrastructure. Use --lisa-dir to run against an already-cloned +LISA checkout instead of cloning; this also allows running LISA tests that have +no [tests.X.lisa.source] configured. ``` azldev image test IMAGE_NAME [flags] @@ -68,7 +64,7 @@ azldev image test IMAGE_NAME [flags] -i, --image-path string Path to the disk image file (resolved from image name if not specified) --junit-xml string Path for writing JUnit XML output --lisa-dir string Path to an already-cloned LISA framework checkout to run against, instead of cloning the framework's configured git source - --test-suite strings Name of a test or test-group to run (may be repeated; defaults to all tests for the image). For images configured with legacy [test-suites], this instead selects legacy test suite names + --test-suite strings Name of a test or test-group to run (may be repeated; defaults to all tests for the image) ``` ### Options inherited from parent commands diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 19abd9722..84445680f 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -14,7 +14,6 @@ All config files share the same schema — there is no distinction between a "ro | `components` | map of objects | Component (package) definitions | [Components](components.md) | | `component-groups` | map of objects | Named groups of components with shared defaults | [Component Groups](component-groups.md) | | `images` | map of objects | Image definitions (VMs, containers) | [Images](images.md) | -| `test-suites` | map of objects | Named test suite definitions referenced by images | [Test Suites](test-suites.md) | | `tests` | map of objects | Named test definitions (new-shape, parse-only) | [Tests and Test Groups](tests.md) | | `test-groups` | map of objects | Named bundles of test references (new-shape, parse-only) | [Tests and Test Groups](tests.md) | | `tools` | object | Configuration for external tools used by azldev | [Tools](tools.md) | diff --git a/docs/user/reference/config/images.md b/docs/user/reference/config/images.md index bc70b269f..edad0140a 100644 --- a/docs/user/reference/config/images.md +++ b/docs/user/reference/config/images.md @@ -39,11 +39,10 @@ The `capabilities` subtable describes what the image supports. All fields are op ## Image Tests -The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section, and/or to entries from the new-shape [`[tests]` / `[test-groups]`](tests.md) sections. +The `tests` subtable links an image to one or more tests or test groups defined in the top-level [`[tests]` / `[test-groups]`](tests.md) sections. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Test Suites | `test-suites` | array of inline tables | No | List of test suite references. Each entry must have a `name` field matching a key in `[test-suites]`. | | Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries (parse-only; see [Tests and Test Groups](tests.md)). | ## Image Publish @@ -90,7 +89,7 @@ description = "Azure-optimized VM image" definition = { type = "kiwi", path = "vm-azure/vm-azure.kiwi", profile = "azure" } ``` -### Image with test suite references +### Image with test references ```toml [images.vm-base] @@ -102,9 +101,9 @@ machine-bootable = true systemd = true [images.vm-base.tests] -test-suites = [ - { name = "smoke" }, - { name = "integration" }, +tests = [ + { name = "static-image-checks" }, + { group = "vm-base-functional" }, ] ``` @@ -122,6 +121,5 @@ channels = ["registry-prod", "registry-staging"] ## Related Resources - [Config File Structure](config-file.md) — top-level config file layout -- [Test Suites](test-suites.md) — test suite definitions - [Tests and Test Groups](tests.md) — new-shape test/group definitions referenced by `[images..tests]` - [Tools](tools.md) — Image Customizer tool configuration diff --git a/docs/user/reference/config/test-suites.md b/docs/user/reference/config/test-suites.md deleted file mode 100644 index 83b9dbfbf..000000000 --- a/docs/user/reference/config/test-suites.md +++ /dev/null @@ -1,171 +0,0 @@ -# Test Suites - -The `[test-suites]` section defines named test suites that can be referenced by images. Each test suite is defined under `[test-suites.]`. - -Test suite names must be simple identifiers (no path separators, traversal segments, or whitespace) since they are used as path components — for example, each pytest suite gets its own Python virtual environment under the project work directory. - -## Test Suite Config - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Description | `description` | string | No | Human-readable description of the test suite | -| Type | `type` | string | Yes | Test framework to use: `"pytest"` or `"lisa"`. | -| Pytest | `pytest` | table | When `type = "pytest"` | Pytest-specific configuration (see below) | -| Lisa | `lisa` | table | Optional (required to run locally) | LISA-specific configuration (see below). May be omitted for metadata-only suites; required to run the suite locally via `azldev image test`. | - -Test suites are referenced by images through the [`[images..tests]`](images.md#image-tests) subtable. Each image can reference one or more test suites by name. - -> **Note:** Each test suite name must be unique across all config files. Defining the same test suite name in two files produces an error. - -## Pytest Suite Config - -When `type = "pytest"`, a `[test-suites..pytest]` subtable must be provided. `azldev` runs the suite by creating (or reusing) a Python virtual environment, installing dependencies, and invoking `python -m pytest` with the configured arguments. - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Working directory | `working-dir` | string | No | Directory used as pytest's CWD. Relative paths are resolved against the config file's directory. Required when `install` is `pyproject` or `requirements`. | -| Test paths | `test-paths` | array of strings | No | Test file paths or directories passed to pytest as positional arguments. Each entry is glob-expanded (including recursive `**`) relative to `working-dir`. Patterns that match nothing are passed through unchanged so pytest reports the failure. | -| Extra args | `extra-args` | array of strings | No | Additional arguments passed to pytest verbatim, after placeholder substitution. See [Placeholders](#placeholders). | -| Install mode | `install` | string | No | How dependencies are installed into the venv. One of `pyproject`, `requirements`, or `none` (default). | - -### Install modes - -| Mode | Behavior | -|------|----------| -| `pyproject` | Installs the project at `working-dir` in editable mode (`pip install -e `). Errors if `pyproject.toml` is not present. | -| `requirements` | Installs from `/requirements.txt`. Errors if the file is not present. | -| `none` (default) | Skips dependency installation entirely. Use when the venv has been pre-populated or pytest is otherwise on `PATH`. | - -`--junit-xml` output requested via the `azldev image test --junit-xml ` CLI flag is appended automatically; you do not need to add it to `extra-args`. Relative `--junit-xml` paths are resolved against the user's current working directory (not the test suite's `working-dir`). - -### Placeholders - -The following placeholders may appear in `extra-args` and are substituted at run time. They are **not** substituted in `test-paths`. - -| Placeholder | Substitution | -|-------------|-------------| -| `{image-path}` | Absolute path to the image artifact under test | -| `{image-name}` | Name of the image being tested | -| `{capabilities}` | Comma-separated list of capability names enabled on the image | - - -## LISA Suite Config - -The `[test-suites..lisa]` subtable is optional. A `type = "lisa"` suite without it is a metadata-only suite (usable by external orchestration but not runnable locally). To run the suite locally via `azldev image test`, the `[test-suites..lisa]` subtable must be provided. - -When run locally, `azldev` executes the suite as follows: it clones the LISA framework at a pinned commit, creates (or reuses) a Python virtual environment and installs the framework into it, generates a runbook from the configured `test-cases`, and boots the image in a QEMU VM to run those cases. VMs are torn down after the run. - -The image under test must already be in **qcow2** format; other formats are rejected. Each run generates and later removes an ephemeral admin SSH key pair for VM access. - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Framework | `framework` | table | Yes | Git source for the LISA framework (see below). | -| Test cases | `test-cases` | array of strings | Yes | LISA test case names to run. They are joined with `\|` into the criteria of the generated runbook. Must be non-empty. | -| Pip pre-install | `pip-pre-install` | array of strings | No | Pip packages installed into the venv *before* the framework, to override framework version pins that conflict with the local environment (e.g., a system-matching `libvirt-python`). | -| Pip extras | `pip-extras` | array of strings | No | Pip extras installed from the framework package, appended as `pip install -e ".[extra1,extra2]"`. | -| Extra args | `extra-args` | array of strings | No | Additional arguments passed to the `lisa` CLI verbatim, after placeholder substitution. See [Placeholders](#placeholders). The generated runbook is always passed via `-r`. | - -### Framework git source - -The `[test-suites..lisa.framework]` subtable pins the LISA framework repository. - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Git URL | `git-url` | string | Yes | URL of the LISA framework git repository. | -| Ref | `ref` | string | Yes | Full 40-character hex commit SHA to check out. Branch names and tags are rejected. | - -The framework checkout is keyed by ref, so updating `ref` clones the new revision. If a pinned ref cannot be checked out in an existing checkout (e.g., it was updated to a newer commit), `azldev` re-clones the repository automatically. - - - -### Basic pytest suite - -```toml -[test-suites.smoke] -description = "Smoke tests for basic image validation" -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests/smoke" -test-paths = ["cases/test_*.py"] -extra-args = ["--image-path", "{image-path}", "--capabilities", "{capabilities}"] -``` - -### Suite with a `requirements.txt` - -```toml -[test-suites.integration] -description = "Integration tests" -type = "pytest" - -[test-suites.integration.pytest] -working-dir = "tests/integration" -install = "requirements" -test-paths = ["**/test_*.py"] -extra-args = ["--image-name", "{image-name}"] -``` - -### Suite that installs from `pyproject.toml` - -```toml -[test-suites.integration-pyproject] -type = "pytest" - -[test-suites.integration-pyproject.pytest] -working-dir = "tests/integration" -install = "pyproject" -test-paths = ["cases/test_*.py"] -``` - -### Suite with no dependency install (default) - -```toml -[test-suites.preinstalled] -type = "pytest" - -[test-suites.preinstalled.pytest] -# install defaults to "none" — pytest must already be available. -test-paths = ["/opt/preinstalled-tests/test_*.py"] -``` - -### LISA suite - -LISA suites are executed locally by `azldev`, which generates a runbook from `test-cases` and boots the image in a QEMU VM. The image must be in qcow2 format. - -```toml -[test-suites.vm-integration] -description = "VM integration tests using LISA" -type = "lisa" - -[test-suites.vm-integration.lisa] -test-cases = ["verify_cpu_count", "verify_grub"] -# Optional: override a framework version pin and pass extra CLI args to LISA. -pip-pre-install = ["libvirt-python==9.0.0"] -extra-args = ["-v"] - -[test-suites.vm-integration.lisa.framework] -git-url = "https://github.com/microsoft/lisa.git" -ref = "abcdef0123456789abcdef0123456789abcdef01" # full 40-char commit SHA -``` - -### Referencing test suites from an image - -```toml -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests/smoke" -test-paths = ["cases/"] - -[images.vm-base] -description = "VM Base Image" - -[images.vm-base.tests] -test-suites = [{ name = "smoke" }] -``` - -## Related Resources - -- [Images](images.md) — image configuration including test references -- [Config File Structure](config-file.md) — top-level config file layout diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md index 6f47dc82d..c043a2b9a 100644 --- a/docs/user/reference/config/tests.md +++ b/docs/user/reference/config/tests.md @@ -32,7 +32,7 @@ can evolve independently. The `[tests..lisa]` subtable is mostly opaque to azldev, but it recognizes a few keys used to select LISA test cases and, optionally, to run the test locally via `azldev image test` (booting the image in a QEMU -VM, same as legacy `[test-suites]` LISA suites). +VM). | Field | TOML Key | Type | Description | |-------|----------|------|-------------| @@ -138,7 +138,6 @@ tests = [ ## Related Resources -- [Test Suites](test-suites.md) - legacy test suite definitions - [Components](components.md#component-tests) — per-component `tests` field - [Images](images.md#image-tests) — per-image `tests` field - [Config File Structure](config-file.md) — top-level config layout diff --git a/internal/app/azldev/agentskill/content/image.md.tmpl b/internal/app/azldev/agentskill/content/image.md.tmpl index ea4ceba0e..b8a7f1b7f 100644 --- a/internal/app/azldev/agentskill/content/image.md.tmpl +++ b/internal/app/azldev/agentskill/content/image.md.tmpl @@ -43,14 +43,14 @@ definition = { type = "kiwi", path = "container-base/container-base.kiwi", profi systemd = false [images.container-base.tests] - test-suites = [{ name = "smoke" }] + tests = [{ name = "smoke" }] ``` - `definition.type` is `kiwi` (the only type today); `path` points at the `.kiwi` file; `profile` selects a kiwi profile (optional). - `capabilities` are tri-state flags describing the image — `machine-bootable`, `container`, `systemd`, `runtime-package-management`. Set only the ones that apply. -- `tests.test-suites` lists the test suites `azldev image test` runs. +- `tests.tests` lists the tests or test groups `azldev image test` runs. - `publish.channels` lists the channels the image publishes to. ## The kiwi definition diff --git a/internal/app/azldev/cmds/image/lisarunner.go b/internal/app/azldev/cmds/image/lisarunner.go index abcaec32d..e78ef1c2f 100644 --- a/internal/app/azldev/cmds/image/lisarunner.go +++ b/internal/app/azldev/cmds/image/lisarunner.go @@ -34,38 +34,6 @@ const ( lisaGeneratedRunbookPrefix = "azldev-generated-" ) -// RunLisaSuite runs a LISA-based test suite by cloning the framework repo, setting up a -// venv, generating a runbook from the configured test cases, and invoking LISA. -func RunLisaSuite( - env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, - imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, -) error { - lisaConfig := suiteConfig.Lisa - if lisaConfig == nil { - return fmt.Errorf( - "test suite %#q of type %#q cannot be run locally: it has no [test-suites.%s.lisa] "+ - "subtable; add one with 'framework' (git-url, ref), 'test-cases', and optional "+ - "'extra-args' to enable local execution", - suiteConfig.Name, projectconfig.TestTypeLisa, suiteConfig.Name, - ) - } - - slog.Info("Running LISA test suite", - slog.String("name", suiteConfig.Name), - slog.String("framework-ref", lisaConfig.Framework.Ref), - slog.Int("test-cases", len(lisaConfig.TestCases)), - slog.String("image-path", options.ImagePath), - ) - - criteria := []lisaCriteria{{Name: strings.Join(lisaConfig.TestCases, "|")}} - - return runLisaLocally( - env, suiteConfig.Name, &lisaConfig.Framework, criteria, - lisaConfig.PipPreInstall, lisaConfig.PipExtras, lisaConfig.ExtraArgs, - imageConfig, options, - ) -} - // RunLisaTestDefinition runs a new-style [tests.X] LISA test definition locally, generating // a runbook from the test's configured criteria and invoking LISA. It requires either // options.LisaDir (an already-cloned LISA checkout) or the test's [tests.X.lisa] subtable @@ -136,10 +104,9 @@ func RunLisaTestDefinition( } // runLisaLocally sets up the LISA framework and venv, generates a runbook from the given -// criteria, and invokes LISA against the given image. It is shared by RunLisaSuite (legacy -// [test-suites] suites) and RunLisaTestDefinition (new-style [tests] definitions). If -// options.LisaDir is set, that checkout is used directly instead of cloning framework; in -// that case framework may be nil. +// criteria, and invokes LISA against the given image. It backs RunLisaTestDefinition +// (new-style [tests] definitions). If options.LisaDir is set, that checkout is used directly +// instead of cloning framework; in that case framework may be nil. func runLisaLocally( env *azldev.Env, name string, framework *projectconfig.GitSourceConfig, criteria []lisaCriteria, pipPreInstall, pipExtras, extraArgs []string, diff --git a/internal/app/azldev/cmds/image/list_test.go b/internal/app/azldev/cmds/image/list_test.go index 6289c08f5..f4c6fc4a8 100644 --- a/internal/app/azldev/cmds/image/list_test.go +++ b/internal/app/azldev/cmds/image/list_test.go @@ -90,7 +90,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Systemd: lo.ToPtr(true), }, Tests: &projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, {Name: "integration"}, }, @@ -106,7 +106,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Container: lo.ToPtr(true), }, Tests: &projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, }, }, @@ -133,7 +133,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, "container", results[0].CapabilitiesSummary) require.NotNil(t, results[0].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}, + Tests: []projectconfig.TestRef{{Name: "smoke"}}, }, *results[0].Tests) assert.Equal(t, "smoke", results[0].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ @@ -157,7 +157,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, "machine-bootable, systemd", results[2].CapabilitiesSummary) require.NotNil(t, results[2].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, + Tests: []projectconfig.TestRef{{Name: "smoke"}, {Name: "integration"}}, }, *results[2].Tests) assert.Equal(t, "smoke, integration", results[2].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 95972e219..5583fbb96 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -8,7 +8,6 @@ import ( "fmt" "log/slog" "path/filepath" - "sort" "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" @@ -16,19 +15,18 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/pelletier/go-toml/v2" - "github.com/samber/lo" "github.com/spf13/cobra" ) // ImageTestOptions holds the options for the 'image test' command. type ImageTestOptions struct { // ImageName is the name of the image (positional argument), used to look up its - // test suites and optionally resolve the image artifact path. + // tests and optionally resolve the image artifact path. ImageName string - // TestSuites optionally selects specific test names or test-group names to run. + // TestSelectors optionally selects specific test names or test-group names to run. // When empty, all tests associated with the image are run. - TestSuites []string + TestSelectors []string // ImagePath is an optional explicit path to the image file. When empty, the image // artifact is resolved from the image name in the output directory. @@ -57,14 +55,11 @@ func NewImageTestCmd() *cobra.Command { Long: `Run tests against an Azure Linux image using test definitions declared in the project configuration. -Images may reference tests directly via [images.NAME.tests.tests] entries, or via -named [test-groups]. Legacy [test-suites] references are still supported. +Images reference tests via [images.NAME.tests.tests] entries, which resolve to +project-level [tests.X] definitions or named [test-groups]. By default, all tests associated with the named image are run. Use --test-suite to select specific test names or test-group names (may be repeated). -For images still configured with legacy [test-suites] (via 'tests.test-suites'), -the values passed to --test-suite are instead matched against those legacy -test suite names. The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -77,14 +72,13 @@ test-paths are expanded automatically. For LISA tests, the test runner executes on the host and boots the image in a QEMU VM. azldev clones the LISA framework, generates a runbook from the test's -configured criteria (or, for legacy [test-suites], its test cases), and runs it -against the image. azldev generates an ephemeral SSH key pair to access the -booted VM and removes it once the test finishes. New-style [tests.X] LISA -definitions require a [tests.X.lisa.source] (git-url, ref) to run locally; -without one, the test is metadata-only and must be run through the LISA -infrastructure. Use --lisa-dir to run against an already-cloned LISA checkout -instead of cloning; this also allows running new-style LISA tests that have no -[tests.X.lisa.source] configured.`, +configured criteria, and runs it against the image. azldev generates an +ephemeral SSH key pair to access the booted VM and removes it once the test +finishes. [tests.X] LISA definitions require a [tests.X.lisa.source] (git-url, +ref) to run locally; without one, the test is metadata-only and must be run +through the LISA infrastructure. Use --lisa-dir to run against an already-cloned +LISA checkout instead of cloning; this also allows running LISA tests that have +no [tests.X.lisa.source] configured.`, Example: ` # Run all tests for an image (artifact auto-resolved from output dir) azldev image test vm-base @@ -108,9 +102,8 @@ instead of cloning; this also allows running new-style LISA tests that have no ValidArgsFunction: generateImageNameCompletions, } - cmd.Flags().StringSliceVar(&options.TestSuites, "test-suite", nil, - "Name of a test or test-group to run (may be repeated; defaults to all tests for the image). "+ - "For images configured with legacy [test-suites], this instead selects legacy test suite names") + cmd.Flags().StringSliceVar(&options.TestSelectors, "test-suite", nil, + "Name of a test or test-group to run (may be repeated; defaults to all tests for the image)") cmd.Flags().StringVarP(&options.ImagePath, "image-path", "i", "", "Path to the disk image file (resolved from image name if not specified)") @@ -140,18 +133,18 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { return err } - resolvedTests, legacySuiteNames, err := resolveImageTestsToRun(cfg, imageConfig, options.TestSuites) + resolvedTests, err := resolveImageTestsToRun(cfg, imageConfig, options.TestSelectors) if err != nil { return err } - if len(resolvedTests) == 0 && len(legacySuiteNames) == 0 { + if len(resolvedTests) == 0 { slog.Warn("No tests to run for image", slog.String("image", options.ImageName)) return nil } - return runImageTests(env, cfg, imageConfig, options, resolvedTests, legacySuiteNames) + return runImageTests(env, imageConfig, options, resolvedTests) } func prepareImageTest(env *azldev.Env, options *ImageTestOptions) (*projectconfig.ImageConfig, error) { @@ -203,11 +196,9 @@ func resolveImageTestPath(env *azldev.Env, options *ImageTestOptions) (string, e func runImageTests( env *azldev.Env, - cfg *projectconfig.ProjectConfig, imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, resolvedTests []projectconfig.ResolvedTest, - legacySuiteNames []string, ) error { var testFailures []string @@ -222,27 +213,9 @@ func runImageTests( } } - for _, suiteName := range legacySuiteNames { - suiteConfig, err := resolveTestSuiteByName(cfg, suiteName) - if err != nil { - return err - } - - if err := runTestSuite(env, suiteConfig, imageConfig, options); err != nil { - slog.Error("Test suite failed", - slog.String("suite", suiteName), - slog.Any("error", err), - ) - - testFailures = append(testFailures, suiteName) - } - } - if len(testFailures) > 0 { - total := len(resolvedTests) + len(legacySuiteNames) - return fmt.Errorf("%d of %d test(s) failed: %s", - len(testFailures), total, strings.Join(testFailures, ", ")) + len(testFailures), len(resolvedTests), strings.Join(testFailures, ", ")) } return nil @@ -252,54 +225,22 @@ func resolveImageTestsToRun( cfg *projectconfig.ProjectConfig, imageConfig *projectconfig.ImageConfig, explicitSelectors []string, -) ([]projectconfig.ResolvedTest, []string, error) { - if imageConfig.Tests != nil && len(imageConfig.Tests.Tests) > 0 { - return resolveCanonicalImageTests(cfg, imageConfig, explicitSelectors) - } - - if len(explicitSelectors) > 0 { - return nil, explicitSelectors, nil - } - - return nil, imageConfig.TestNames(), nil -} - -func resolveCanonicalImageTests( - cfg *projectconfig.ProjectConfig, - imageConfig *projectconfig.ImageConfig, - explicitSelectors []string, -) ([]projectconfig.ResolvedTest, []string, error) { - warnIfLegacyTestSuitesIgnored(imageConfig) - +) ([]projectconfig.ResolvedTest, error) { if len(explicitSelectors) > 0 { resolvedTests, err := cfg.ResolveTestSelectors(explicitSelectors) if err != nil { - return nil, nil, fmt.Errorf("resolve test selectors: %w", err) + return nil, fmt.Errorf("resolve test selectors: %w", err) } - return resolvedTests, nil, nil + return resolvedTests, nil } resolvedTests, err := cfg.ResolveImageTests(imageConfig) if err != nil { - return nil, nil, fmt.Errorf("resolve image tests: %w", err) - } - - return resolvedTests, nil, nil -} - -func warnIfLegacyTestSuitesIgnored(imageConfig *projectconfig.ImageConfig) { - if len(imageConfig.Tests.TestSuites) == 0 { - return + return nil, fmt.Errorf("resolve image tests: %w", err) } - slog.Warn( - "image defines both 'tests.tests' and legacy 'tests.test-suites'; "+ - "the legacy 'tests.test-suites' entries are ignored and only 'tests.tests' "+ - "will run. Remove 'tests.test-suites' to silence this warning", - slog.String("image", imageConfig.Name), - slog.Int("ignored-test-suites", len(imageConfig.Tests.TestSuites)), - ) + return resolvedTests, nil } func runResolvedTest( @@ -370,46 +311,6 @@ func decodePytestConfig(raw map[string]any) (*projectconfig.PytestConfig, error) return pytestConfig, nil } -// resolveTestSuiteByName looks up a test suite by name in the project configuration. -func resolveTestSuiteByName( - cfg *projectconfig.ProjectConfig, suiteName string, -) (*projectconfig.TestSuiteConfig, error) { - suiteConfig, ok := cfg.TestSuites[suiteName] - if !ok { - availableSuites := lo.Keys(cfg.TestSuites) - sort.Strings(availableSuites) - - if len(availableSuites) == 0 { - return nil, fmt.Errorf( - "test suite %#q not found; no test suites defined in project configuration", suiteName) - } - - return nil, fmt.Errorf( - "test suite %#q not found; available test suites: %s", - suiteName, strings.Join(availableSuites, ", "), - ) - } - - return &suiteConfig, nil -} - -// runTestSuite dispatches a single test suite to the appropriate runner. -func runTestSuite( - env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, - imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, -) error { - switch suiteConfig.Type { - case projectconfig.TestTypePytest: - return RunPytestSuite(env, suiteConfig, imageConfig, options) - - case projectconfig.TestTypeLisa: - return RunLisaSuite(env, suiteConfig, imageConfig, options) - - default: - return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) - } -} - // validateFileExists returns an error if the path does not point to an existing regular file. func validateFileExists(fs opctx.FS, path string) error { isDir, err := fileutils.DirExists(fs, path) diff --git a/internal/app/azldev/cmds/image/test_internal_test.go b/internal/app/azldev/cmds/image/test_internal_test.go index 5cd3deb55..051f13f8d 100644 --- a/internal/app/azldev/cmds/image/test_internal_test.go +++ b/internal/app/azldev/cmds/image/test_internal_test.go @@ -34,61 +34,13 @@ func TestResolveImageTestsToRun_UsesNewTestsRefs(t *testing.T) { }, } - resolved, legacy, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) + resolved, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) require.NoError(t, err) - assert.Empty(t, legacy) require.Len(t, resolved, 2) assert.Equal(t, "static-image-checks", resolved[0].Name) assert.Equal(t, "functional_core", resolved[1].Name) } -func TestResolveImageTestsToRun_FallsBackToLegacyTestSuites(t *testing.T) { - testEnv := testutils.NewTestEnv(t) - imageCfg := &projectconfig.ImageConfig{ - Tests: &projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, - }, - } - - resolved, legacy, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) - require.NoError(t, err) - assert.Empty(t, resolved) - assert.Equal(t, []string{"smoke", "integration"}, legacy) -} - -func TestResolveImageTestsToRun_WarnsWhenBothTestsAndLegacyTestSuitesPresent(t *testing.T) { - var buf bytes.Buffer - - prev := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))) - - t.Cleanup(func() { slog.SetDefault(prev) }) - - testEnv := testutils.NewTestEnv(t) - testEnv.Config.Tests = map[string]projectconfig.TestDefinition{ - "static-image-checks": {Type: "pytest", Pytest: map[string]any{"working-dir": "/project/tests"}}, - } - - imageCfg := &projectconfig.ImageConfig{ - Name: "vm-base", - Tests: &projectconfig.ImageTestsConfig{ - Tests: []projectconfig.TestRef{{Name: "static-image-checks"}}, - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}, - }, - } - - resolved, legacy, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) - require.NoError(t, err) - assert.Empty(t, legacy) - require.Len(t, resolved, 1) - assert.Equal(t, "static-image-checks", resolved[0].Name) - - logs := buf.String() - assert.Contains(t, logs, "tests.test-suites") - assert.Contains(t, logs, "ignored") - assert.Contains(t, logs, "vm-base") -} - func TestResolveImageTestsToRun_NoWarnWhenOnlyNewTestsPresent(t *testing.T) { var buf bytes.Buffer @@ -109,7 +61,7 @@ func TestResolveImageTestsToRun_NoWarnWhenOnlyNewTestsPresent(t *testing.T) { }, } - _, _, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) + _, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) require.NoError(t, err) assert.NotContains(t, buf.String(), "ignored") } diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 4c2a38ce8..bf3a25fa8 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -63,9 +63,6 @@ type ConfigFile struct { // to be applied to sets of binary packages. PackageGroups map[string]PackageGroupConfig `toml:"package-groups,omitempty" validate:"dive" jsonschema:"title=Package groups,description=Definitions of package groups for shared binary package configuration"` - // Definitions of test suites. - TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Test Suites,description=Definitions of test suites for this project"` - // Definitions of individual tests (new schema, [tests.X]). Tests map[string]TestDefinition `toml:"tests,omitempty" validate:"dive" jsonschema:"title=Tests,description=Definitions of individual tests"` @@ -142,10 +139,6 @@ func (f ConfigFile) Validate() error { } } - if err := validateTestSuites(f.TestSuites); err != nil { - return err - } - if err := validateTestDefinitions(f.Tests); err != nil { return err } @@ -153,25 +146,6 @@ func (f ConfigFile) Validate() error { return nil } -func validateTestSuites(testSuites map[string]TestSuiteConfig) error { - for suiteName, suite := range testSuites { - // Suite names are used as path components (e.g., for the per-suite venv directory), - // so reject anything that could escape the intended directory or otherwise be unsafe - // across platforms. - if err := fileutils.ValidateFilename(suiteName); err != nil { - return fmt.Errorf("invalid test suite name %#q:\n%w", suiteName, err) - } - - suite.Name = suiteName - - if err := suite.Validate(); err != nil { - return fmt.Errorf("invalid test suite %#q:\n%w", suiteName, err) - } - } - - return nil -} - func validateTestDefinitions(tests map[string]TestDefinition) error { for testName, testDef := range tests { // Test names are used as path components (e.g., for the per-test venv directory), diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index 55b19be37..0f89e4a34 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -358,35 +358,6 @@ func TestProjectConfigValidation_NonContradictingImageCapabilities(t *testing.T) require.NoError(t, err) } -func TestProjectConfigValidation_LegacyTestSuitesEmitsDeprecationWarning(t *testing.T) { - var buf bytes.Buffer - - prev := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))) - - t.Cleanup(func() { slog.SetDefault(prev) }) - - cfg := projectconfig.NewProjectConfig() - cfg.TestSuites = map[string]projectconfig.TestSuiteConfig{ - "static-image-checks": {}, - } - cfg.Images = map[string]projectconfig.ImageConfig{ - "legacy-img": { - Tests: &projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "static-image-checks"}}, - }, - }, - } - - err := cfg.Validate() - require.NoError(t, err) - - logs := buf.String() - assert.Contains(t, logs, "deprecated") - assert.Contains(t, logs, "tests.test-suites") - assert.Contains(t, logs, "legacy-img") -} - func TestProjectConfigValidation_NewShapeTestsNoDeprecationWarning(t *testing.T) { var buf bytes.Buffer diff --git a/internal/projectconfig/image.go b/internal/projectconfig/image.go index bd9db48b0..566f36542 100644 --- a/internal/projectconfig/image.go +++ b/internal/projectconfig/image.go @@ -161,33 +161,28 @@ func (c *ImageCapabilities) EnabledNames() []string { // ImageTestsConfig holds the test-related configuration for an image. type ImageTestsConfig struct { - // TestSuites is the list of test suite references that apply to this image. Each - // reference identifies a test suite defined in the top-level [test-suites] section - // and may carry per-test metadata in the future (e.g., required vs optional). - TestSuites []TestSuiteRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=List of test suite references that apply to this image"` - - // Tests is the new-shape list of test or test-group references that apply to this + // Tests is the list of test or test-group references that apply to this // image. References must resolve to entries in the project-level [tests] or // [test-groups] maps; resolution is the responsibility of the test layer. Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this image"` } -// TestSuiteRef is a reference to a named test suite. Using a structured type (rather than -// a bare string) allows per-test metadata to be added later without a breaking config change. -type TestSuiteRef struct { - // Name is the key into the top-level [test-suites] map. - Name string `toml:"name" json:"name" jsonschema:"required,title=Name,description=Name of the test suite (must match a key in [test-suites])"` -} - -// TestNames returns the test suite names referenced by this image. +// TestNames returns the test and test-group reference labels for this image, for +// display/summary purposes. Group references are prefixed with "group:". func (i *ImageConfig) TestNames() []string { if i.Tests == nil { return nil } - names := make([]string, len(i.Tests.TestSuites)) - for idx, ref := range i.Tests.TestSuites { - names[idx] = ref.Name + names := make([]string, 0, len(i.Tests.Tests)) + + for _, ref := range i.Tests.Tests { + switch { + case ref.Name != "": + names = append(names, ref.Name) + case ref.Group != "": + names = append(names, "group:"+ref.Group) + } } return names diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 0b831e931..feb2ef959 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -42,7 +42,6 @@ func loadAndResolveProjectConfig( Distros: make(map[string]DistroDefinition), GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), - TestSuites: make(map[string]TestSuiteConfig), Tests: make(map[string]TestDefinition), TestGroups: make(map[string]TestGroup), } @@ -147,10 +146,6 @@ func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return err } - if err := mergeTestSuites(resolvedCfg, loadedCfg); err != nil { - return err - } - if err := mergeTests(resolvedCfg, loadedCfg); err != nil { return err } @@ -327,24 +322,6 @@ func mergePackageGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error return nil } -// mergeTestSuites merges test suite definitions from a loaded config file into the -// resolved config. Duplicate test suite names are not allowed. -func mergeTestSuites(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { - for suiteName, suite := range loadedCfg.TestSuites { - if _, ok := resolvedCfg.TestSuites[suiteName]; ok { - return fmt.Errorf("%w: test suite %#q", ErrDuplicateTestSuites, suiteName) - } - - // Fill out fields not explicitly serialized. - suite.Name = suiteName - suite.SourceConfigFile = loadedCfg - - resolvedCfg.TestSuites[suiteName] = *(suite.WithAbsolutePaths(loadedCfg.dir)) - } - - return nil -} - // mergeTests merges individual test definitions from a loaded config file into the // resolved config. Duplicate test names are not allowed. func mergeTests(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index c84f87ae1..7b369ea5c 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -1222,166 +1222,6 @@ rpm-channel = "devel" } } -func TestLoadAndResolveProjectConfig_TestSuite(t *testing.T) { - const configContents = ` -[test-suites.smoke] -type = "pytest" -description = "Smoke tests for images" - -[test-suites.smoke.pytest] -working-dir = "tests" -test-paths = ["cases/test_*.py"] -extra-args = ["--image-path", "{image-path}"] - -[test-suites.integration] -type = "lisa" -description = "LISA integration tests" - -[test-suites.integration.lisa] -test-cases = ["verify_cpu_count", "verify_grub"] -extra-args = ["-v", "qcow2:{image-path}"] - -[test-suites.integration.lisa.framework] -git-url = "https://github.com/microsoft/lisa.git" -ref = "abcdef0123456789abcdef0123456789abcdef01" -` - - configDir := filepath.Dir(testConfigPath) - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.NoError(t, err) - - require.Len(t, config.TestSuites, 2) - - // Check pytest test. - if assert.Contains(t, config.TestSuites, "smoke") { - smokeTest := config.TestSuites["smoke"] - assert.Equal(t, "smoke", smokeTest.Name) - assert.Equal(t, TestTypePytest, smokeTest.Type) - assert.Equal(t, "Smoke tests for images", smokeTest.Description) - require.NotNil(t, smokeTest.Pytest) - assert.Equal(t, filepath.Join(configDir, "tests"), smokeTest.Pytest.WorkingDir) - assert.Equal(t, []string{"cases/test_*.py"}, smokeTest.Pytest.TestPaths) - assert.Equal(t, []string{"--image-path", "{image-path}"}, smokeTest.Pytest.ExtraArgs) - } - - // Check LISA test. - if assert.Contains(t, config.TestSuites, "integration") { - lisaTest := config.TestSuites["integration"] - assert.Equal(t, "integration", lisaTest.Name) - assert.Equal(t, TestTypeLisa, lisaTest.Type) - assert.Equal(t, "LISA integration tests", lisaTest.Description) - require.NotNil(t, lisaTest.Lisa) - assert.Equal(t, "https://github.com/microsoft/lisa.git", lisaTest.Lisa.Framework.GitURL) - assert.Equal(t, "abcdef0123456789abcdef0123456789abcdef01", lisaTest.Lisa.Framework.Ref) - assert.Equal(t, []string{"verify_cpu_count", "verify_grub"}, lisaTest.Lisa.TestCases) - assert.Equal(t, []string{"-v", "qcow2:{image-path}"}, lisaTest.Lisa.ExtraArgs) - } -} - -func TestLoadAndResolveProjectConfig_DuplicateTests(t *testing.T) { - testFiles := []struct { - path string - contents string - }{ - {testConfigPath, ` -includes = ["include.toml"] - -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests" -test-paths = ["cases/"] -`}, - {"/project/include.toml", ` -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests" -test-paths = ["other/"] -`}, - } - - ctx := testctx.NewCtx() - - for _, testFile := range testFiles { - require.NoError(t, fileutils.MkdirAll(ctx.FS(), filepath.Dir(testFile.path))) - require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) - } - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) - require.ErrorIs(t, err, ErrDuplicateTestSuites) -} - -func TestLoadAndResolveProjectConfig_InvalidTestType(t *testing.T) { - const configContents = ` -[test-suites.bad] -type = "unsupported" -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - assert.ErrorIs(t, err, ErrUnknownTestType) -} - -func TestLoadAndResolveProjectConfig_TestMissingRequiredField(t *testing.T) { - const configContents = ` -[test-suites.smoke] -type = "pytest" -# Missing [test-suites.smoke.pytest] subtable -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - assert.ErrorIs(t, err, ErrMissingTestField) -} - -func TestLoadAndResolveProjectConfig_TestSuiteMissingType(t *testing.T) { - const configContents = ` -[test-suites.smoke] -# 'type' intentionally omitted. -description = "no type set" -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - require.ErrorIs(t, err, ErrMissingTestField) - assert.Contains(t, err.Error(), "type") -} - -func TestLoadAndResolveProjectConfig_TestSuiteInvalidName(t *testing.T) { - // Names containing path separators or traversal segments must be rejected at - // config load time since they are used as path components (e.g., venv directories). - const configContents = ` -[test-suites."../escape"] -type = "pytest" - -[test-suites."../escape".pytest] -working-dir = "tests" -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - assert.Contains(t, err.Error(), "invalid test suite name") -} - func TestLoadAndResolveProjectConfig_TestInvalidName(t *testing.T) { // Names containing path separators or traversal segments must be rejected at // config load time since they are used as path components (e.g., venv directories). @@ -1402,52 +1242,6 @@ test-paths = ["test_smoke.py"] assert.Contains(t, err.Error(), "invalid test name") } -func TestLoadAndResolveProjectConfig_ImageWithValidTestRef(t *testing.T) { - const configContents = ` -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests" -test-paths = ["cases/"] - -[images.myimage] -description = "Test image" - -[images.myimage.tests] -test-suites = [{ name = "smoke" }] -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.NoError(t, err) - - if assert.Contains(t, config.Images, "myimage") { - require.NotNil(t, config.Images["myimage"].Tests) - assert.Equal(t, []TestSuiteRef{{Name: "smoke"}}, config.Images["myimage"].Tests.TestSuites) - } -} - -func TestLoadAndResolveProjectConfig_ImageWithInvalidTestRef(t *testing.T) { - const configContents = ` -[images.myimage] -description = "Test image" - -[images.myimage.tests] -test-suites = [{ name = "nonexistent" }] -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - require.ErrorIs(t, err, ErrUndefinedTestSuite) - assert.Contains(t, err.Error(), "nonexistent") -} - func TestLoadAndResolveProjectConfig_ImageCapabilities_FipsEnabledAndCVM(t *testing.T) { const configContents = ` [images.myimage] @@ -1594,48 +1388,6 @@ rpm-channel = "new-channel" "rpm-channel should take precedence over the deprecated channel field") } -func TestLoadAndResolveProjectConfig_TestSuiteInstallMode(t *testing.T) { - const configContents = ` -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests" -install = "requirements" -test-paths = ["cases/"] -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.NoError(t, err) - - if assert.Contains(t, config.TestSuites, "smoke") { - smokeTest := config.TestSuites["smoke"] - require.NotNil(t, smokeTest.Pytest) - assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.Install) - assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.EffectiveInstallMode()) - } -} - -func TestLoadAndResolveProjectConfig_TestSuiteInvalidInstallMode(t *testing.T) { - const configContents = ` -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -install = "invalid" -` - - ctx := testctx.NewCtx() - require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) - require.Error(t, err) - assert.ErrorIs(t, err, ErrInvalidInstallMode) -} - func TestLoadAndResolveProjectConfig_CircularInclude(t *testing.T) { t.Run("direct self-include", func(t *testing.T) { ctx := testctx.NewCtx() diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index 1788e9f0a..f7005b746 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -6,7 +6,6 @@ package projectconfig import ( "errors" "fmt" - "log/slog" "sort" "strings" @@ -46,9 +45,6 @@ type ProjectConfig struct { // Definitions of package groups with shared configuration. PackageGroups map[string]PackageGroupConfig `toml:"package-groups,omitempty" json:"packageGroups,omitempty" jsonschema:"title=Package groups,description=Mapping of package group names to configurations for publish-time routing"` - // Definitions of test suites. - TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=Mapping of test suite names to configurations"` - // Definitions of individual tests. Tests map[string]TestDefinition `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Mapping of test names to configurations"` @@ -72,7 +68,6 @@ func NewProjectConfig() ProjectConfig { Resources: ResourcesConfig{RpmRepos: make(map[string]RpmRepoResource)}, GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), - TestSuites: make(map[string]TestSuiteConfig), Tests: make(map[string]TestDefinition), TestGroups: make(map[string]TestGroup), } @@ -93,10 +88,6 @@ func (cfg *ProjectConfig) Validate() error { return err } - if err := validateImageTestReferences(cfg.Images, cfg.TestSuites); err != nil { - return err - } - if err := validateImageCapabilities(cfg.Images); err != nil { return err } @@ -286,33 +277,6 @@ func validatePackageGroupMembership(groups map[string]PackageGroupConfig) error return nil } -// validateImageTestReferences checks that every test suite name in an image's -// [ImageConfig.Tests.TestSuites] list corresponds to a defined entry in the top-level -// TestSuites map. The legacy [tests.test-suites] image key is deprecated in favor of the -// new [tests.tests] shape; a warning is emitted for each image still using it. -func validateImageTestReferences(images map[string]ImageConfig, testSuites map[string]TestSuiteConfig) error { - for imageName, image := range images { - if image.Tests != nil && len(image.Tests.TestSuites) > 0 { - slog.Warn( - "image uses deprecated 'tests.test-suites' key; migrate to 'tests.tests' "+ - "(referencing [tests.X] / [test-groups.X]) as legacy test-suites will be removed", - slog.String("image", imageName), - ) - } - - for _, suiteName := range image.TestNames() { - if _, ok := testSuites[suiteName]; !ok { - return fmt.Errorf( - "%w: image %#q references test suite %#q, which is not defined in [test-suites]", - ErrUndefinedTestSuite, imageName, suiteName, - ) - } - } - } - - return nil -} - // validateImageCapabilities enforces mutual exclusivity among delivery-kind // capability flags. At most one of machine-bootable, container, wsl, and // installer-media may be true for a single image. diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index c795c0c5e..da93d9e51 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -7,8 +7,6 @@ import ( "encoding/hex" "errors" "fmt" - - "dario.cat/mergo" ) // TestType indicates the type of test framework used to run a test suite. @@ -22,8 +20,6 @@ const ( ) var ( - // ErrDuplicateTestSuites is returned when duplicate conflicting test suite definitions are found. - ErrDuplicateTestSuites = errors.New("duplicate test suite") // ErrDuplicateTests is returned when duplicate conflicting [tests] entries are found. ErrDuplicateTests = errors.New("duplicate test") // ErrDuplicateTestGroups is returned when duplicate conflicting [test-groups] entries are found. @@ -32,8 +28,6 @@ var ( ErrUnknownTestType = errors.New("unknown test type") // ErrMissingTestField is returned when a required test config field is missing. ErrMissingTestField = errors.New("missing required test field") - // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. - ErrUndefinedTestSuite = errors.New("undefined test suite reference") // ErrUndefinedTest is returned when a test reference points to a missing [tests] entry. ErrUndefinedTest = errors.New("undefined test reference") // ErrUndefinedTestGroup is returned when a test reference points to a missing [test-groups] entry. @@ -81,10 +75,6 @@ type TestSuiteConfig struct { // Lisa holds LISA-specific configuration. Optional for a "lisa" suite; when present // it drives generation and execution of a LISA runbook. Lisa *LisaConfig `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration used to generate and run a LISA runbook (optional)"` - - // Reference to the source config file that this definition came from; not present - // in serialized files. - SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` } // PytestInstallMode specifies how Python dependencies are installed for a pytest suite. @@ -283,45 +273,3 @@ func (m PytestInstallMode) isValid() bool { return false } } - -// MergeUpdatesFrom updates the test suite config with overrides present in other. -func (t *TestSuiteConfig) MergeUpdatesFrom(other *TestSuiteConfig) error { - err := mergo.Merge(t, other, mergo.WithOverride, mergo.WithAppendSlice) - if err != nil { - return fmt.Errorf("failed to merge test suite config:\n%w", err) - } - - return nil -} - -// WithAbsolutePaths returns a copy of the test suite config with relative file paths converted -// to absolute paths (relative to referenceDir). -func (t *TestSuiteConfig) WithAbsolutePaths(referenceDir string) *TestSuiteConfig { - result := &TestSuiteConfig{ - Name: t.Name, - Description: t.Description, - Type: t.Type, - SourceConfigFile: t.SourceConfigFile, - } - - if t.Pytest != nil { - result.Pytest = &PytestConfig{ - WorkingDir: makeAbsolute(referenceDir, t.Pytest.WorkingDir), - TestPaths: append([]string(nil), t.Pytest.TestPaths...), - ExtraArgs: append([]string(nil), t.Pytest.ExtraArgs...), - Install: t.Pytest.Install, - } - } - - if t.Lisa != nil { - result.Lisa = &LisaConfig{ - Framework: t.Lisa.Framework, - TestCases: append([]string(nil), t.Lisa.TestCases...), - PipPreInstall: append([]string(nil), t.Lisa.PipPreInstall...), - PipExtras: append([]string(nil), t.Lisa.PipExtras...), - ExtraArgs: append([]string(nil), t.Lisa.ExtraArgs...), - } - } - - return result -} diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index fe0a28f93..5cd93de9a 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -70,13 +70,13 @@ func TestImageConfig_TestNames(t *testing.T) { t.Run("with tests", func(t *testing.T) { img := projectconfig.ImageConfig{ Tests: &projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, - {Name: "integration"}, + {Group: "integration"}, }, }, } - assert.Equal(t, []string{"smoke", "integration"}, img.TestNames()) + assert.Equal(t, []string{"smoke", "group:integration"}, img.TestNames()) }) t.Run("no tests returns empty", func(t *testing.T) { @@ -320,102 +320,3 @@ func TestPytestConfig_EffectiveInstallMode(t *testing.T) { assert.Equal(t, projectconfig.PytestInstallRequirements, cfg.EffectiveInstallMode()) }) } - -func TestTestSuiteConfig_MergeUpdatesFrom(t *testing.T) { - t.Run("merge overrides non-zero fields", func(t *testing.T) { - base := projectconfig.TestSuiteConfig{ - Name: "smoke", - Type: projectconfig.TestTypePytest, - Pytest: &projectconfig.PytestConfig{ - WorkingDir: "tests", - }, - } - other := projectconfig.TestSuiteConfig{ - Description: "Updated description", - } - require.NoError(t, base.MergeUpdatesFrom(&other)) - assert.Equal(t, "Updated description", base.Description) - assert.Equal(t, "tests", base.Pytest.WorkingDir) - }) - - t.Run("merge appends test-paths", func(t *testing.T) { - base := projectconfig.TestSuiteConfig{ - Name: "smoke", - Type: projectconfig.TestTypePytest, - Pytest: &projectconfig.PytestConfig{ - TestPaths: []string{"cases/"}, - }, - } - other := projectconfig.TestSuiteConfig{ - Pytest: &projectconfig.PytestConfig{ - TestPaths: []string{"extra/"}, - }, - } - require.NoError(t, base.MergeUpdatesFrom(&other)) - assert.Equal(t, []string{"cases/", "extra/"}, base.Pytest.TestPaths) - }) -} - -func TestValidateTestSuiteReferences(t *testing.T) { - t.Run("valid references", func(t *testing.T) { - cfg := projectconfig.ProjectConfig{ - Images: map[string]projectconfig.ImageConfig{ - "myimage": { - Name: "myimage", - Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, - }, - }, - TestSuites: map[string]projectconfig.TestSuiteConfig{ - "smoke": { - Name: "smoke", - Type: projectconfig.TestTypePytest, - Pytest: &projectconfig.PytestConfig{ - WorkingDir: "tests", - }, - }, - }, - Components: make(map[string]projectconfig.ComponentConfig), - ComponentGroups: make(map[string]projectconfig.ComponentGroupConfig), - Distros: make(map[string]projectconfig.DistroDefinition), - GroupsByComponent: make(map[string][]string), - PackageGroups: make(map[string]projectconfig.PackageGroupConfig), - } - assert.NoError(t, cfg.Validate()) - }) - - t.Run("undefined test reference", func(t *testing.T) { - cfg := projectconfig.ProjectConfig{ - Images: map[string]projectconfig.ImageConfig{ - "myimage": { - Name: "myimage", - Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, - }, - }, - TestSuites: make(map[string]projectconfig.TestSuiteConfig), - Components: make(map[string]projectconfig.ComponentConfig), - ComponentGroups: make(map[string]projectconfig.ComponentGroupConfig), - Distros: make(map[string]projectconfig.DistroDefinition), - GroupsByComponent: make(map[string][]string), - PackageGroups: make(map[string]projectconfig.PackageGroupConfig), - } - err := cfg.Validate() - require.Error(t, err) - require.ErrorIs(t, err, projectconfig.ErrUndefinedTestSuite) - assert.Contains(t, err.Error(), "nonexistent") - }) - - t.Run("image with no tests is valid", func(t *testing.T) { - cfg := projectconfig.ProjectConfig{ - Images: map[string]projectconfig.ImageConfig{ - "myimage": {Name: "myimage"}, - }, - TestSuites: make(map[string]projectconfig.TestSuiteConfig), - Components: make(map[string]projectconfig.ComponentConfig), - ComponentGroups: make(map[string]projectconfig.ComponentGroupConfig), - Distros: make(map[string]projectconfig.DistroDefinition), - GroupsByComponent: make(map[string][]string), - PackageGroups: make(map[string]projectconfig.PackageGroupConfig), - } - assert.NoError(t, cfg.Validate()) - }) -} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 974e5d428..27564d867 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -443,14 +443,6 @@ "title": "Package groups", "description": "Definitions of package groups for shared binary package configuration" }, - "test-suites": { - "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" - }, - "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" - }, "tests": { "additionalProperties": { "$ref": "#/$defs/TestDefinition" @@ -637,26 +629,6 @@ "additionalProperties": false, "type": "object" }, - "GitSourceConfig": { - "properties": { - "git-url": { - "type": "string", - "title": "Git URL", - "description": "URL of the git repository" - }, - "ref": { - "type": "string", - "title": "Ref", - "description": "Commit SHA to check out (full hex hash)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "git-url", - "ref" - ] - }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -782,14 +754,6 @@ }, "ImageTestsConfig": { "properties": { - "test-suites": { - "items": { - "$ref": "#/$defs/TestSuiteRef" - }, - "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" - }, "tests": { "items": { "$ref": "#/$defs/TestRef" @@ -802,53 +766,6 @@ "additionalProperties": false, "type": "object" }, - "LisaConfig": { - "properties": { - "framework": { - "$ref": "#/$defs/GitSourceConfig", - "title": "Framework", - "description": "Git source for the LISA framework" - }, - "test-cases": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test cases", - "description": "LISA test case names to run; combined into a generated runbook's criteria" - }, - "pip-pre-install": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip pre-install", - "description": "Pip packages to install before the framework (for overriding version pins)" - }, - "pip-extras": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip extras", - "description": "Pip extras to install from the LISA framework package" - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "framework", - "test-cases" - ] - }, "Origin": { "properties": { "type": { @@ -1080,43 +997,6 @@ "additionalProperties": false, "type": "object" }, - "PytestConfig": { - "properties": { - "working-dir": { - "type": "string", - "title": "Working directory", - "description": "Directory to use as CWD when running pytest" - }, - "test-paths": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test paths", - "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." - }, - "install": { - "type": "string", - "enum": [ - "pyproject", - "requirements", - "none" - ], - "title": "Install mode", - "description": "How to install Python dependencies: pyproject, requirements, or none (default)" - } - }, - "additionalProperties": false, - "type": "object" - }, "ReleaseConfig": { "properties": { "calculation": { @@ -1682,53 +1562,6 @@ "additionalProperties": false, "type": "object" }, - "TestSuiteConfig": { - "properties": { - "description": { - "type": "string", - "title": "Description", - "description": "Description of this test suite" - }, - "type": { - "type": "string", - "enum": [ - "pytest", - "lisa" - ], - "title": "Type", - "description": "Type of test framework (pytest or lisa)" - }, - "pytest": { - "$ref": "#/$defs/PytestConfig", - "title": "Pytest config", - "description": "Pytest-specific configuration (required when type is pytest)" - }, - "lisa": { - "$ref": "#/$defs/LisaConfig", - "title": "LISA config", - "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type" - ] - }, - "TestSuiteRef": { - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] - }, "ToolsConfig": { "properties": { "imageCustomizer": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 974e5d428..27564d867 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -443,14 +443,6 @@ "title": "Package groups", "description": "Definitions of package groups for shared binary package configuration" }, - "test-suites": { - "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" - }, - "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" - }, "tests": { "additionalProperties": { "$ref": "#/$defs/TestDefinition" @@ -637,26 +629,6 @@ "additionalProperties": false, "type": "object" }, - "GitSourceConfig": { - "properties": { - "git-url": { - "type": "string", - "title": "Git URL", - "description": "URL of the git repository" - }, - "ref": { - "type": "string", - "title": "Ref", - "description": "Commit SHA to check out (full hex hash)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "git-url", - "ref" - ] - }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -782,14 +754,6 @@ }, "ImageTestsConfig": { "properties": { - "test-suites": { - "items": { - "$ref": "#/$defs/TestSuiteRef" - }, - "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" - }, "tests": { "items": { "$ref": "#/$defs/TestRef" @@ -802,53 +766,6 @@ "additionalProperties": false, "type": "object" }, - "LisaConfig": { - "properties": { - "framework": { - "$ref": "#/$defs/GitSourceConfig", - "title": "Framework", - "description": "Git source for the LISA framework" - }, - "test-cases": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test cases", - "description": "LISA test case names to run; combined into a generated runbook's criteria" - }, - "pip-pre-install": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip pre-install", - "description": "Pip packages to install before the framework (for overriding version pins)" - }, - "pip-extras": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip extras", - "description": "Pip extras to install from the LISA framework package" - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "framework", - "test-cases" - ] - }, "Origin": { "properties": { "type": { @@ -1080,43 +997,6 @@ "additionalProperties": false, "type": "object" }, - "PytestConfig": { - "properties": { - "working-dir": { - "type": "string", - "title": "Working directory", - "description": "Directory to use as CWD when running pytest" - }, - "test-paths": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test paths", - "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." - }, - "install": { - "type": "string", - "enum": [ - "pyproject", - "requirements", - "none" - ], - "title": "Install mode", - "description": "How to install Python dependencies: pyproject, requirements, or none (default)" - } - }, - "additionalProperties": false, - "type": "object" - }, "ReleaseConfig": { "properties": { "calculation": { @@ -1682,53 +1562,6 @@ "additionalProperties": false, "type": "object" }, - "TestSuiteConfig": { - "properties": { - "description": { - "type": "string", - "title": "Description", - "description": "Description of this test suite" - }, - "type": { - "type": "string", - "enum": [ - "pytest", - "lisa" - ], - "title": "Type", - "description": "Type of test framework (pytest or lisa)" - }, - "pytest": { - "$ref": "#/$defs/PytestConfig", - "title": "Pytest config", - "description": "Pytest-specific configuration (required when type is pytest)" - }, - "lisa": { - "$ref": "#/$defs/LisaConfig", - "title": "LISA config", - "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type" - ] - }, - "TestSuiteRef": { - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] - }, "ToolsConfig": { "properties": { "imageCustomizer": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 974e5d428..27564d867 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -443,14 +443,6 @@ "title": "Package groups", "description": "Definitions of package groups for shared binary package configuration" }, - "test-suites": { - "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" - }, - "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" - }, "tests": { "additionalProperties": { "$ref": "#/$defs/TestDefinition" @@ -637,26 +629,6 @@ "additionalProperties": false, "type": "object" }, - "GitSourceConfig": { - "properties": { - "git-url": { - "type": "string", - "title": "Git URL", - "description": "URL of the git repository" - }, - "ref": { - "type": "string", - "title": "Ref", - "description": "Commit SHA to check out (full hex hash)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "git-url", - "ref" - ] - }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -782,14 +754,6 @@ }, "ImageTestsConfig": { "properties": { - "test-suites": { - "items": { - "$ref": "#/$defs/TestSuiteRef" - }, - "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" - }, "tests": { "items": { "$ref": "#/$defs/TestRef" @@ -802,53 +766,6 @@ "additionalProperties": false, "type": "object" }, - "LisaConfig": { - "properties": { - "framework": { - "$ref": "#/$defs/GitSourceConfig", - "title": "Framework", - "description": "Git source for the LISA framework" - }, - "test-cases": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test cases", - "description": "LISA test case names to run; combined into a generated runbook's criteria" - }, - "pip-pre-install": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip pre-install", - "description": "Pip packages to install before the framework (for overriding version pins)" - }, - "pip-extras": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Pip extras", - "description": "Pip extras to install from the LISA framework package" - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "framework", - "test-cases" - ] - }, "Origin": { "properties": { "type": { @@ -1080,43 +997,6 @@ "additionalProperties": false, "type": "object" }, - "PytestConfig": { - "properties": { - "working-dir": { - "type": "string", - "title": "Working directory", - "description": "Directory to use as CWD when running pytest" - }, - "test-paths": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Test paths", - "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." - }, - "extra-args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Extra arguments", - "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." - }, - "install": { - "type": "string", - "enum": [ - "pyproject", - "requirements", - "none" - ], - "title": "Install mode", - "description": "How to install Python dependencies: pyproject, requirements, or none (default)" - } - }, - "additionalProperties": false, - "type": "object" - }, "ReleaseConfig": { "properties": { "calculation": { @@ -1682,53 +1562,6 @@ "additionalProperties": false, "type": "object" }, - "TestSuiteConfig": { - "properties": { - "description": { - "type": "string", - "title": "Description", - "description": "Description of this test suite" - }, - "type": { - "type": "string", - "enum": [ - "pytest", - "lisa" - ], - "title": "Type", - "description": "Type of test framework (pytest or lisa)" - }, - "pytest": { - "$ref": "#/$defs/PytestConfig", - "title": "Pytest config", - "description": "Pytest-specific configuration (required when type is pytest)" - }, - "lisa": { - "$ref": "#/$defs/LisaConfig", - "title": "LISA config", - "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type" - ] - }, - "TestSuiteRef": { - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] - }, "ToolsConfig": { "properties": { "imageCustomizer": { From 1deeda37f22423dce8e055f2a1a1b6420b1c64c9 Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Wed, 9 Sep 2026 17:30:04 +0530 Subject: [PATCH 2/2] fix(config): preserve pytest working-dir as authored instead of absolutizing at load --- internal/app/azldev/cmds/image/test.go | 4 ++++ internal/projectconfig/loader_test.go | 26 ++++++++++++++++++++++++ internal/projectconfig/tests.go | 28 ++++++++++++++++++++------ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 5583fbb96..8848c2925 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -279,6 +279,10 @@ func testDefinitionToSuiteConfig(resolvedTest projectconfig.ResolvedTest) (*proj return nil, fmt.Errorf("decode pytest config for test %#q:\n%w", resolvedTest.Name, err) } + // The stored config keeps 'working-dir' as authored; resolve it relative to + // the defining config file's directory only here, for execution. + pytestConfig.WorkingDir = resolvedTest.Definition.PytestWorkingDir() + suiteConfig := &projectconfig.TestSuiteConfig{ Name: resolvedTest.Name, Description: resolvedTest.Definition.Description, diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index 7b369ea5c..bf7f86d50 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -1388,6 +1388,32 @@ rpm-channel = "new-channel" "rpm-channel should take precedence over the deprecated channel field") } +func TestLoadAndResolveProjectConfig_PytestWorkingDirPreservedAsAuthored(t *testing.T) { + const configContents = ` +[tests.smoke] +type = "pytest" + +[tests.smoke.pytest] +working-dir = "tests" +test-paths = ["cases/"] +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) + + config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.NoError(t, err) + + require.Contains(t, config.Tests, "smoke") + // working-dir must be preserved exactly as authored, not rewritten to an + // absolute path at load/dump time. + assert.Equal(t, "tests", config.Tests["smoke"].Pytest["working-dir"]) + // ...but it resolves to an absolute path relative to the defining config + // file's directory at execution time (via recorded provenance), so relative + // paths in included files remain correct. + assert.Equal(t, "/project/tests", config.Tests["smoke"].PytestWorkingDir()) +} + func TestLoadAndResolveProjectConfig_CircularInclude(t *testing.T) { t.Run("direct self-include", func(t *testing.T) { ctx := testctx.NewCtx() diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go index a2eec25ec..7f4dc0e56 100644 --- a/internal/projectconfig/tests.go +++ b/internal/projectconfig/tests.go @@ -81,23 +81,39 @@ type TestDefinition struct { Lisa map[string]any `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration"` Tmt map[string]any `toml:"tmt,omitempty" json:"tmt,omitempty" jsonschema:"title=TMT config,description=TMT-specific configuration"` Pytest map[string]any `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=pytest-specific configuration"` + + // dir is the directory of the config file that defined this test. It is not + // serialized (so 'azldev config dump' is unaffected) and is populated during + // config load to resolve relative pytest paths at test-execution time. + dir string } -// WithAbsolutePaths returns a copy of the test definition with any relative -// paths in framework-specific subtables converted to absolute paths. +// WithAbsolutePaths returns a deep copy of the test definition, recording +// referenceDir (the defining config file's directory) so relative pytest paths +// (e.g. 'working-dir') can be resolved at test-execution time. Path values in +// framework-specific subtables are preserved exactly as authored, so +// 'azldev config dump' shows the user-defined values; see +// [TestDefinition.PytestWorkingDir] for the execution-time resolution. func (t TestDefinition) WithAbsolutePaths(referenceDir string) TestDefinition { result := t + result.dir = referenceDir result.Lisa = cloneStringAnyMap(t.Lisa) result.Tmt = cloneStringAnyMap(t.Tmt) result.Pytest = cloneStringAnyMap(t.Pytest) - if workingDir, ok := result.Pytest["working-dir"].(string); ok { - result.Pytest["working-dir"] = makeAbsolute(referenceDir, workingDir) - } - return result } +// PytestWorkingDir returns the pytest 'working-dir' resolved to an absolute path +// relative to the config file that defined the test. The stored config keeps the +// authored (possibly relative) value; this resolves it only for execution. An +// empty 'working-dir' returns "". +func (t TestDefinition) PytestWorkingDir() string { + workingDir, _ := t.Pytest["working-dir"].(string) + + return makeAbsolute(t.dir, workingDir) +} + // TestGroup is a [test-groups.X] declaration: a named bundle of test references that // images or components can target via a single name. type TestGroup struct {