From aa7ae88e5f437d05174af037cbcd63c015d85709 Mon Sep 17 00:00:00 2001 From: reuben olinsky Date: Tue, 9 Jun 2026 10:10:34 -0700 Subject: [PATCH] feat(projectconfig): add typed tests, test-groups, and component test refs Rework the test-suites configuration into a richer test model while keeping the on-disk `test-suites` / `testSuites` wire names unchanged, so existing config files and `config dump` consumers are not broken (the config-key rename is intentionally deferred to a separate change). - Test groups: named bundles of tests referenceable from images/components. - Per-component test references, in addition to per-image references. - A closed `kind` enum (functional/performance) and a `long-running` hint. - Project-level validation that image/component/group references resolve, plus tightened JSON schema constraints (exactly-one ref shape; per-type required subtable) for editor-time feedback. - `azldev image test` drives the new model. - Reference docs and regenerated JSON schema / CLI docs / snapshots. In-code identifiers use the new "test" naming; only the TOML key (`test-suites`) and JSON dump key (`testSuites`) retain the legacy names to avoid a breaking change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: bhagyapathak --- docs/user/reference/cli/azldev_image_test.md | 33 +- docs/user/reference/config/components.md | 19 + docs/user/reference/config/config-file.md | 3 +- docs/user/reference/config/images.md | 12 +- docs/user/reference/config/test-suites.md | 133 ------ docs/user/reference/config/tests.md | 112 +++++ go.mod | 2 +- internal/app/azldev/cmds/image/list.go | 22 +- internal/app/azldev/cmds/image/list_test.go | 10 +- .../app/azldev/cmds/image/pytestrunner.go | 20 +- .../azldev/cmds/image/pytestrunner_test.go | 6 +- internal/app/azldev/cmds/image/test.go | 181 +++++--- internal/app/azldev/cmds/image/test_test.go | 2 +- internal/projectconfig/component.go | 27 ++ internal/projectconfig/configfile.go | 60 ++- internal/projectconfig/configfile_test.go | 78 ++++ internal/projectconfig/fingerprint_test.go | 5 + internal/projectconfig/image.go | 55 ++- internal/projectconfig/loader.go | 45 +- internal/projectconfig/loader_test.go | 28 +- internal/projectconfig/project.go | 140 +++++- internal/projectconfig/test.go | 419 ++++++++++++++++++ .../{testsuite_test.go => test_test.go} | 99 +++-- internal/projectconfig/testgroup.go | 85 ++++ internal/projectconfig/testsuite.go | 206 --------- ...ainer_config_generate-schema_stdout_1.snap | 152 ++++++- ...shots_config_generate-schema_stdout_1.snap | 152 ++++++- schemas/azldev.schema.json | 152 ++++++- 28 files changed, 1659 insertions(+), 599 deletions(-) delete mode 100644 docs/user/reference/config/test-suites.md create mode 100644 docs/user/reference/config/tests.md create mode 100644 internal/projectconfig/test.go rename internal/projectconfig/{testsuite_test.go => test_test.go} (78%) create mode 100644 internal/projectconfig/testgroup.go delete mode 100644 internal/projectconfig/testsuite.go diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 89c29c12..5c880002 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -6,15 +6,16 @@ Run tests against an Azure Linux image ### Synopsis -Run tests against an Azure Linux image using test suites defined in the +Run tests against an Azure Linux image using tests defined in the project configuration. -Test suites are defined in the [test-suites] section of azldev.toml and referenced -by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +Tests are defined in the [tests] section of azldev.toml and referenced by images +via the [images.NAME.tests] subtable, either directly by name or through a +test-group. Each test specifies a type and framework-specific configuration in +a matching subtable. -By default, all test suites associated with the named image are run. Use ---test-suite to select specific suites (may be repeated). +By default, all tests associated with the named image (directly or via a +test-group) are run. Use --test to select specific tests (may be repeated). The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -32,17 +33,17 @@ azldev image test IMAGE_NAME [flags] ### Examples ``` - # Run all test suites for an image (artifact auto-resolved from output dir) + # Run all tests for an image (artifact auto-resolved from output dir) azldev image test vm-base - # Run all test suites with an explicit image path + # Run all tests with an explicit image path azldev image test vm-base --image-path ./out/images/vm-base/image.raw - # Run a specific test suite - azldev image test vm-base --test-suite common-vm-checks + # Run a specific test + azldev image test vm-base --test common-vm-checks - # Run multiple specific test suites - azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks + # Run multiple specific tests + azldev image test vm-base --test common-vm-checks --test vm-base-checks # Generate JUnit XML output azldev image test vm-base --junit-xml results.xml @@ -51,10 +52,10 @@ azldev image test IMAGE_NAME [flags] ### Options ``` - -h, --help help for test - -i, --image-path string Path to the disk image file (resolved from image name if not specified) - --junit-xml string Path for writing JUnit XML output - --test-suite strings Name of a test suite to run (may be repeated; defaults to all suites for the image) + -h, --help help for test + -i, --image-path string Path to the disk image file (resolved from image name if not specified) + --junit-xml string Path for writing JUnit XML output + --test strings Name of a test to run (may be repeated; defaults to all tests for the image) ``` ### Options inherited from parent commands diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 35344a8d..b77e8a40 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -16,6 +16,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | +| Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (see [Tests](tests.md)) | ### Bare Components @@ -300,6 +301,24 @@ rpm-channel = "rpm-devel" rpm-channel = "none" ``` +## Component Tests + +The `[components..tests]` subtable lists test or test-group references that apply to this component. Each entry is a [TestRef](tests.md#test-reference) with exactly one of `name` or `group` set. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Tests | `test-suites` | array of [TestRef](tests.md#test-reference) | No | References to `[test-suites.]` entries or `[test-groups.]` entries | + +```toml +[components.kernel.tests] +test-suites = [ + { group = "kernel-bvt" }, + { name = "kdump-smoke" }, +] +``` + +Test associations are metadata and never participate in fingerprints — adding, removing, or renaming a test reference does not invalidate a component's build. + ## Source File References The `[[components..source-files]]` array defines additional source files that azldev should download before building. These are files not available in the dist-git repository or lookaside cache — typically binaries, pre-built artifacts, or files from custom hosting. diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index e1ae30bb..5819126b 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -14,7 +14,8 @@ All config files share the same schema — there is no distinction between a "ro | `components` | map of objects | Component (package) definitions | [Components](components.md) | | `component-groups` | map of objects | Named groups of components with shared defaults | [Component Groups](component-groups.md) | | `images` | map of objects | Image definitions (VMs, containers) | [Images](images.md) | -| `test-suites` | map of objects | Named test suite definitions referenced by images | [Test Suites](test-suites.md) | +| `test-suites` | map of objects | Named test definitions referenced by images and components | [Tests](tests.md) | +| `test-groups` | map of objects | Named bundles of test references | [Tests](tests.md#test-group) | | `tools` | object | Configuration for external tools used by azldev | [Tools](tools.md) | | `default-package-config` | object | Project-wide default applied to all binary packages | [Package Groups — Resolution Order](package-groups.md#resolution-order) | | `package-groups` | map of objects | Named groups of binary packages with shared config | [Package Groups](package-groups.md) | diff --git a/docs/user/reference/config/images.md b/docs/user/reference/config/images.md index ebbd5e53..dea1feab 100644 --- a/docs/user/reference/config/images.md +++ b/docs/user/reference/config/images.md @@ -35,11 +35,11 @@ The `capabilities` subtable describes what the image supports. All fields are op ## Image Tests -The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section. +The `tests` subtable links an image to tests and test-groups defined in the top-level [`[test-suites]` / `[test-groups]`](tests.md) sections. Each entry is a [TestRef](tests.md#test-reference) with exactly one of `name` or `group` set. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Test Suites | `test-suites` | array of inline tables | No | List of test suite references. Each entry must have a `name` field matching a key in `[test-suites]`. | +| Tests | `test-suites` | array of [TestRef](tests.md#test-reference) | No | References to `[test-suites.]` entries or `[test-groups.]` entries | ## Image Publish @@ -85,7 +85,7 @@ description = "Azure-optimized VM image" definition = { type = "kiwi", path = "vm-azure/vm-azure.kiwi", profile = "azure" } ``` -### Image with test suite references +### Image with test references ```toml [images.vm-base] @@ -98,8 +98,8 @@ systemd = true [images.vm-base.tests] test-suites = [ - { name = "smoke" }, - { name = "integration" }, + { group = "smoke" }, + { name = "integration-boot" }, ] ``` @@ -117,5 +117,5 @@ channels = ["registry-prod", "registry-staging"] ## Related Resources - [Config File Structure](config-file.md) — top-level config file layout -- [Test Suites](test-suites.md) — test suite definitions +- [Tests](tests.md) — test and test-group definitions referenced by `[images..tests]` - [Tools](tools.md) — Image Customizer tool configuration diff --git a/docs/user/reference/config/test-suites.md b/docs/user/reference/config/test-suites.md deleted file mode 100644 index 83f50e39..00000000 --- a/docs/user/reference/config/test-suites.md +++ /dev/null @@ -1,133 +0,0 @@ -# Test Suites - -The `[test-suites]` section defines named test suites that can be referenced by images. Each test suite is defined under `[test-suites.]`. - -Test suite names must be simple identifiers (no path separators, traversal segments, or whitespace) since they are used as path components — for example, each pytest suite gets its own Python virtual environment under the project work directory. - -## Test Suite Config - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Description | `description` | string | No | Human-readable description of the test suite | -| Type | `type` | string | Yes | Test framework to use: `"pytest"` or `"lisa"`. | -| Pytest | `pytest` | table | When `type = "pytest"` | Pytest-specific configuration (see below) | - -Test suites are referenced by images through the [`[images..tests]`](images.md#image-tests) subtable. Each image can reference one or more test suites by name. - -> **Note:** Each test suite name must be unique across all config files. Defining the same test suite name in two files produces an error. - -## Pytest Suite Config - -When `type = "pytest"`, a `[test-suites..pytest]` subtable must be provided. `azldev` runs the suite by creating (or reusing) a Python virtual environment, installing dependencies, and invoking `python -m pytest` with the configured arguments. - -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Working directory | `working-dir` | string | No | Directory used as pytest's CWD. Relative paths are resolved against the config file's directory. Required when `install` is `pyproject` or `requirements`. | -| Test paths | `test-paths` | array of strings | No | Test file paths or directories passed to pytest as positional arguments. Each entry is glob-expanded (including recursive `**`) relative to `working-dir`. Patterns that match nothing are passed through unchanged so pytest reports the failure. | -| Extra args | `extra-args` | array of strings | No | Additional arguments passed to pytest verbatim, after placeholder substitution. See [Placeholders](#placeholders). | -| Install mode | `install` | string | No | How dependencies are installed into the venv. One of `pyproject`, `requirements`, or `none` (default). | - -### Install modes - -| Mode | Behavior | -|------|----------| -| `pyproject` | Installs the project at `working-dir` in editable mode (`pip install -e `). Errors if `pyproject.toml` is not present. | -| `requirements` | Installs from `/requirements.txt`. Errors if the file is not present. | -| `none` (default) | Skips dependency installation entirely. Use when the venv has been pre-populated or pytest is otherwise on `PATH`. | - -`--junit-xml` output requested via the `azldev image test --junit-xml ` CLI flag is appended automatically; you do not need to add it to `extra-args`. Relative `--junit-xml` paths are resolved against the user's current working directory (not the test suite's `working-dir`). - -### Placeholders - -The following placeholders may appear in `extra-args` and are substituted at run time. They are **not** substituted in `test-paths`. - -| Placeholder | Substitution | -|-------------|-------------| -| `{image-path}` | Absolute path to the image artifact under test | -| `{image-name}` | Name of the image being tested | -| `{capabilities}` | Comma-separated list of capability names enabled on the image | - - -## Examples - -### Basic pytest suite - -```toml -[test-suites.smoke] -description = "Smoke tests for basic image validation" -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests/smoke" -test-paths = ["cases/test_*.py"] -extra-args = ["--image-path", "{image-path}", "--capabilities", "{capabilities}"] -``` - -### Suite with a `requirements.txt` - -```toml -[test-suites.integration] -description = "Integration tests" -type = "pytest" - -[test-suites.integration.pytest] -working-dir = "tests/integration" -install = "requirements" -test-paths = ["**/test_*.py"] -extra-args = ["--image-name", "{image-name}"] -``` - -### Suite that installs from `pyproject.toml` - -```toml -[test-suites.integration-pyproject] -type = "pytest" - -[test-suites.integration-pyproject.pytest] -working-dir = "tests/integration" -install = "pyproject" -test-paths = ["cases/test_*.py"] -``` - -### Suite with no dependency install (default) - -```toml -[test-suites.preinstalled] -type = "pytest" - -[test-suites.preinstalled.pytest] -# install defaults to "none" — pytest must already be available. -test-paths = ["/opt/preinstalled-tests/test_*.py"] -``` - -### LISA suite (external) - -LISA test suites are metadata-only — they are not executed by `azldev` but are used by external orchestration systems. - -```toml -[test-suites.vm-integration] -description = "VM integration tests using LISA" -type = "lisa" -``` - -### Referencing test suites from an image - -```toml -[test-suites.smoke] -type = "pytest" - -[test-suites.smoke.pytest] -working-dir = "tests/smoke" -test-paths = ["cases/"] - -[images.vm-base] -description = "VM Base Image" - -[images.vm-base.tests] -test-suites = [{ name = "smoke" }] -``` - -## Related Resources - -- [Images](images.md) — image configuration including test references -- [Config File Structure](config-file.md) — top-level config file layout diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md new file mode 100644 index 00000000..265c482d --- /dev/null +++ b/docs/user/reference/config/tests.md @@ -0,0 +1,112 @@ +# Tests and Test Groups + +The `[test-suites]` section defines named, framework-specific test configurations that components and images can target by name. The `[test-groups]` section bundles those tests under a single name so callers can reference a curated set without enumerating every member. + +A `TestRef` is the structured reference used by image and component `tests` subtables. Each ref points at exactly one of a `[test-suites.]` entry or a `[test-groups.]` entry. + +> Test references on images and components are metadata and do **not** participate in build fingerprints — adding or removing a reference never invalidates a component's build. + +## Test Config + +Each entry under `[test-suites.]` describes one configuration of one runner. Framework-specific options live in a typed subtable (`pytest`) whose schema is validated by azldev. `lisa` tests are accepted as a type but have no subtable — they are metadata for external orchestration and are not run by `azldev image test`. + +Test names must be simple identifiers (no path separators, traversal segments, or whitespace) since they may be used as path components (for example, each pytest test gets its own Python virtual environment under the project work directory). + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Type | `type` | string | **Yes** | Test framework: `pytest` or `lisa` | +| Description | `description` | string | No | Human-readable description | +| Kind | `kind` | string array | No | Closed enum of classification hints: `functional`, `performance`. May be multi-valued. | +| Long running | `long-running` | bool | No | Hints that this test may run for hours. Declarative cost hint, not a configurable timeout. | +| Pytest | `pytest` | [PytestConfig](#pytest-config) | When `type = "pytest"` | pytest-specific configuration | + +The framework subtable must match the declared `type`: `type = "pytest"` requires a `pytest` subtable; `type = "lisa"` forbids it. + +> **Note:** Each test name must be unique across all config files. Defining the same test name in two files produces an error. + +### Pytest Config + +When `type = "pytest"`, a `[test-suites..pytest]` subtable must be provided. `azldev` runs the test by creating (or reusing) a Python virtual environment, optionally installing dependencies, and invoking `python -m pytest` with the configured arguments. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Working directory | `working-dir` | string | No | Directory used as pytest's CWD. Relative paths are resolved against the config file's directory. Required when `install` is `pyproject` or `requirements`. | +| Test paths | `test-paths` | array of strings | No | Test file paths or directories passed to pytest as positional arguments. Each entry is glob-expanded (including recursive `**`) relative to `working-dir`. Patterns that match nothing are passed through so pytest reports the failure. | +| Extra args | `extra-args` | array of strings | No | Additional arguments passed to pytest verbatim, after placeholder substitution. Use `{image-path}` as a placeholder for the image path. | +| Install mode | `install` | string | No | One of `pyproject`, `requirements`, or `none` (default). | + +## Test Group + +Each entry under `[test-groups.]` names an ordered list of test references that callers can target as a single unit. Group membership is one layer of indirection: groups list **test names only**, never other groups. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Description | `description` | string | No | Human-readable description | +| Tests | `tests` | string array | No | Ordered list of test names (keys in `[test-suites]`) that belong to this group | + +Every name in `tests` must resolve to a defined `[test-suites.]` entry; nested group references are not supported. + +> **Note:** Each test-group name must be unique across all config files. Defining the same group name in two files produces an error. + +## Test Reference + +`TestRef` is an inline table used by `[images..tests.test-suites]` and `[components..tests.test-suites]`. Exactly one of `name` or `group` must be set: + +| Field | TOML Key | Type | Description | +|-------|----------|------|-------------| +| Name | `name` | string | References a `[test-suites.]` entry | +| Group | `group` | string | References a `[test-groups.]` entry | + +References must resolve at project load time — an unknown test or group is a config error. + +## Example + +```toml +[test-suites.bvt-ssh] +type = "pytest" +description = "Basic SSH boot verification" +kind = ["functional"] + +[test-suites.bvt-ssh.pytest] +working-dir = "tests/bvt" +test-paths = ["test_ssh.py"] +extra-args = ["--image-path", "{image-path}"] +install = "pyproject" + +[test-suites.kdump-smoke] +type = "pytest" +description = "Smoke test for kdump" + +[test-suites.kdump-smoke.pytest] +working-dir = "tests/kdump" +test-paths = ["test_kdump.py"] + +[test-suites.kdump-lisa] +type = "lisa" +description = "Kdump scenarios driven by external LISA orchestration" +long-running = true + +[test-groups.bvt] +description = "Build verification tests" +tests = ["bvt-ssh", "kdump-smoke"] + +[images.vm-base.tests] +test-suites = [{ group = "bvt" }] + +[components.kernel.tests] +test-suites = [ + { name = "kdump-smoke" }, + { name = "kdump-lisa" }, +] +``` + +## Running tests + +Use [`azldev image test IMAGE_NAME`](../cli/azldev_image_test.md) to run all `pytest` tests associated with an image (directly or via test-groups). Use `--test ` to run a single named test. `lisa` tests are recognized by the schema but are not currently executed by `azldev image test`. + +## Related Resources + +- [Components](components.md#component-tests) — per-component `test-suites` references +- [Images](images.md#image-tests) — per-image `test-suites` references +- [Config File Structure](config-file.md) — top-level config layout +- [`azldev image test`](../cli/azldev_image_test.md) — CLI reference for running tests against an image diff --git a/go.mod b/go.mod index b3eaa94d..a3c97d6b 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/muesli/termenv v0.16.0 github.com/nxadm/tail v1.4.11 github.com/opencontainers/selinux v1.15.1 + github.com/pb33f/ordered-map/v2 v2.3.1 github.com/pelletier/go-toml/v2 v2.4.2 github.com/pmezard/go-difflib v1.0.0 github.com/samber/lo v1.53.0 @@ -131,7 +132,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/internal/app/azldev/cmds/image/list.go b/internal/app/azldev/cmds/image/list.go index d0d6bc78..d36ce0ab 100644 --- a/internal/app/azldev/cmds/image/list.go +++ b/internal/app/azldev/cmds/image/list.go @@ -40,7 +40,8 @@ type ImageListResult struct { // structure. Tests projectconfig.ImageTestsConfig `json:"tests" table:"-"` - // TestsSummary is a comma-separated summary of test suite names for table display. + // TestsSummary is a comma-separated summary of test refs (test names and + // "@group" markers) for table display. TestsSummary string `json:"-" table:"Tests"` // Publish holds the publish settings for this image. @@ -136,7 +137,7 @@ func ListImages(env *azldev.Env, options *ListImageOptions) ([]ImageListResult, Capabilities: imageConfig.Capabilities, CapabilitiesSummary: strings.Join(imageConfig.Capabilities.EnabledNames(), ", "), Tests: imageConfig.Tests, - TestsSummary: strings.Join(imageConfig.TestNames(), ", "), + TestsSummary: strings.Join(formatTestRefs(imageConfig.Tests.Tests), ", "), Publish: imageConfig.Publish, PublishSummary: strings.Join(imageConfig.Publish.Channels, ", "), Definition: ImageDefinitionResult{ @@ -149,6 +150,23 @@ func ListImages(env *azldev.Env, options *ListImageOptions) ([]ImageListResult, return results, nil } +// formatTestRefs renders a slice of [projectconfig.TestRef] for display in +// "image list" table output. Direct test refs are emitted as bare names; group +// refs are emitted with an "@" prefix to distinguish them. Order is preserved. +func formatTestRefs(refs []projectconfig.TestRef) []string { + out := make([]string, 0, len(refs)) + for _, ref := range refs { + switch { + case ref.Name != "": + out = append(out, ref.Name) + case ref.Group != "": + out = append(out, "@"+ref.Group) + } + } + + return out +} + // matchesAnyPattern returns true if name matches any of the given glob patterns. func matchesAnyPattern(name string, patterns []string) (bool, error) { for _, pattern := range patterns { diff --git a/internal/app/azldev/cmds/image/list_test.go b/internal/app/azldev/cmds/image/list_test.go index 6c572ac3..3d0f5b8c 100644 --- a/internal/app/azldev/cmds/image/list_test.go +++ b/internal/app/azldev/cmds/image/list_test.go @@ -90,7 +90,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Systemd: lo.ToPtr(true), }, Tests: projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, {Name: "integration"}, }, @@ -106,7 +106,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Container: lo.ToPtr(true), }, Tests: projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, }, }, @@ -132,7 +132,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Nil(t, results[0].Capabilities.MachineBootable) assert.Equal(t, "container", results[0].CapabilitiesSummary) assert.Equal(t, projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}, + Tests: []projectconfig.TestRef{{Name: "smoke"}}, }, results[0].Tests) assert.Equal(t, "smoke", results[0].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ @@ -144,7 +144,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Nil(t, results[1].Capabilities.MachineBootable) assert.Nil(t, results[1].Capabilities.Container) assert.Empty(t, results[1].CapabilitiesSummary) - assert.Empty(t, results[1].Tests.TestSuites) + assert.Empty(t, results[1].Tests.Tests) assert.Empty(t, results[1].TestsSummary) assert.Empty(t, results[1].Publish.Channels) assert.Empty(t, results[1].PublishSummary) @@ -155,7 +155,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Nil(t, results[2].Capabilities.Container) assert.Equal(t, "machine-bootable, systemd", results[2].CapabilitiesSummary) assert.Equal(t, projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, + Tests: []projectconfig.TestRef{{Name: "smoke"}, {Name: "integration"}}, }, results[2].Tests) assert.Equal(t, "smoke, integration", results[2].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ diff --git a/internal/app/azldev/cmds/image/pytestrunner.go b/internal/app/azldev/cmds/image/pytestrunner.go index a3227fdc..4ffee346 100644 --- a/internal/app/azldev/cmds/image/pytestrunner.go +++ b/internal/app/azldev/cmds/image/pytestrunner.go @@ -34,18 +34,18 @@ const ( capabilitiesPlaceholder = "{capabilities}" ) -// RunPytestSuite runs a pytest-based test suite natively using a Python venv. -func RunPytestSuite( - env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, +// RunPytest runs a pytest-based test natively using a Python venv. +func RunPytest( + env *azldev.Env, testConfig *projectconfig.TestConfig, imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, ) error { - pytestConfig := suiteConfig.Pytest + pytestConfig := testConfig.Pytest if pytestConfig == nil { - return fmt.Errorf("test suite %#q is missing pytest configuration", suiteConfig.Name) + return fmt.Errorf("test %#q is missing pytest configuration", testConfig.Name) } - slog.Info("Running pytest test suite", - slog.String("name", suiteConfig.Name), + slog.Info("Running pytest test", + slog.String("name", testConfig.Name), slog.String("working-dir", pytestConfig.WorkingDir), slog.String("image-path", options.ImagePath), ) @@ -68,7 +68,7 @@ func RunPytestSuite( } // Set up or reuse the venv. - venvDir, err := ensurePytestVenv(env, suiteConfig.Name, pytestConfig) + venvDir, err := ensurePytestVenv(env, testConfig.Name, pytestConfig) if err != nil { return err } @@ -103,7 +103,7 @@ func RunPytestSuite( return nil } -// ensurePytestVenv creates or reuses a Python venv for the given test suite and installs +// ensurePytestVenv creates or reuses a Python venv for the given test and installs // dependencies according to the configured install mode. The venv is created under the // project's work directory. func ensurePytestVenv( @@ -115,7 +115,7 @@ func ensurePytestVenv( // 'pytest-venv/...' tree under the user's CWD, which is surprising and can // pollute repos. Fail fast instead. return "", fmt.Errorf( - "cannot create pytest venv for test suite %#q: project work directory is not configured", + "cannot create pytest venv for test %#q: project work directory is not configured", testName, ) } diff --git a/internal/app/azldev/cmds/image/pytestrunner_test.go b/internal/app/azldev/cmds/image/pytestrunner_test.go index d48db0cf..1cc98e1f 100644 --- a/internal/app/azldev/cmds/image/pytestrunner_test.go +++ b/internal/app/azldev/cmds/image/pytestrunner_test.go @@ -218,8 +218,8 @@ func TestBuildNativePytestArgs_CapabilitiesEmpty(t *testing.T) { assert.Equal(t, []string{"--capabilities", "container"}, args) } -func TestRunPytestSuite_MissingPytestConfig(t *testing.T) { - suiteConfig := &projectconfig.TestSuiteConfig{ +func TestRunPytest_MissingPytestConfig(t *testing.T) { + testConfig := &projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, } @@ -228,7 +228,7 @@ func TestRunPytestSuite_MissingPytestConfig(t *testing.T) { ImagePath: "/images/test.raw", } - err := image.RunPytestSuite(nil, suiteConfig, testImageConfig(), options) + err := image.RunPytest(nil, testConfig, testImageConfig(), options) require.Error(t, err) assert.Contains(t, err.Error(), "missing pytest configuration") } diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 9ea9ad54..fb78c228 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -23,12 +23,12 @@ import ( // ImageTestOptions holds the options for the 'image test' command. type ImageTestOptions struct { // ImageName is the name of the image (positional argument), used to look up its - // test suites and optionally resolve the image artifact path. + // tests and optionally resolve the image artifact path. ImageName string - // TestSuites optionally selects specific test suites to run. When empty, all test - // suites associated with the image are run. - TestSuites []string + // Tests optionally selects specific tests to run. When empty, all tests + // associated with the image (directly or via test-groups) are run. + Tests []string // ImagePath is an optional explicit path to the image file. When empty, the image // artifact is resolved from the image name in the output directory. @@ -49,15 +49,16 @@ func NewImageTestCmd() *cobra.Command { cmd := &cobra.Command{ Use: "test IMAGE_NAME", Short: "Run tests against an Azure Linux image", - Long: `Run tests against an Azure Linux image using test suites defined in the + Long: `Run tests against an Azure Linux image using tests defined in the project configuration. -Test suites are defined in the [test-suites] section of azldev.toml and referenced -by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +Tests are defined in the [tests] section of azldev.toml and referenced by images +via the [images.NAME.tests] subtable, either directly by name or through a +test-group. Each test specifies a type and framework-specific configuration in +a matching subtable. -By default, all test suites associated with the named image are run. Use ---test-suite to select specific suites (may be repeated). +By default, all tests associated with the named image (directly or via a +test-group) are run. Use --test to select specific tests (may be repeated). The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -67,17 +68,17 @@ dependencies from pyproject.toml in the working directory, and runs pytest with the configured test paths and extra arguments. Use {image-path} in extra-args to insert the image path. Glob patterns (including **) in test-paths are expanded automatically.`, - Example: ` # Run all test suites for an image (artifact auto-resolved from output dir) + Example: ` # Run all tests for an image (artifact auto-resolved from output dir) azldev image test vm-base - # Run all test suites with an explicit image path + # Run all tests with an explicit image path azldev image test vm-base --image-path ./out/images/vm-base/image.raw - # Run a specific test suite - azldev image test vm-base --test-suite common-vm-checks + # Run a specific test + azldev image test vm-base --test common-vm-checks - # Run multiple specific test suites - azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks + # Run multiple specific tests + azldev image test vm-base --test common-vm-checks --test vm-base-checks # Generate JUnit XML output azldev image test vm-base --junit-xml results.xml`, @@ -90,8 +91,8 @@ test-paths are expanded automatically.`, ValidArgsFunction: generateImageNameCompletions, } - cmd.Flags().StringSliceVar(&options.TestSuites, "test-suite", nil, - "Name of a test suite to run (may be repeated; defaults to all suites for the image)") + cmd.Flags().StringSliceVar(&options.Tests, "test", nil, + "Name of a test to run (may be repeated; defaults to all tests for the image)") cmd.Flags().StringVarP(&options.ImagePath, "image-path", "i", "", "Path to the disk image file (resolved from image name if not specified)") @@ -104,7 +105,7 @@ test-paths are expanded automatically.`, return cmd } -// runImageTest resolves which test suites to run and dispatches each one. +// runImageTest resolves which tests to run and dispatches each one. func runImageTest(env *azldev.Env, options *ImageTestOptions) error { cfg := env.Config() if cfg == nil { @@ -143,7 +144,7 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { // Absolutize JUnitXMLPath against the user's CWD so pytest writes to the location the // user expected — pytest itself resolves relative paths against its own working - // directory (the test suite's working-dir), which is rarely what the user intended. + // directory (the test's working-dir), which is rarely what the user intended. if options.JUnitXMLPath != "" && !filepath.IsAbs(options.JUnitXMLPath) { absJUnitPath, err := filepath.Abs(options.JUnitXMLPath) if err != nil { @@ -153,116 +154,150 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { options.JUnitXMLPath = absJUnitPath } - // Determine which test suites to run. - suiteNames := resolveTestSuiteNames(imageConfig, options.TestSuites) + // Determine which tests to run. + imageTestNames := expandImageTestRefs(imageConfig, cfg) + testNames := resolveTestNames(imageTestNames, options.Tests) - // Warn when explicitly requested suites are not referenced by the image config. - if len(options.TestSuites) > 0 { - warnUnassociatedSuites(options.ImageName, imageConfig, options.TestSuites) + // Warn when explicitly requested tests are not referenced by the image config. + if len(options.Tests) > 0 { + warnUnassociatedTests(options.ImageName, imageTestNames, options.Tests) } - if len(suiteNames) == 0 { - slog.Warn("No test suites to run for image", slog.String("image", options.ImageName)) + if len(testNames) == 0 { + slog.Warn("No tests to run for image", slog.String("image", options.ImageName)) return nil } - // Resolve and run each test suite, continuing past failures so all suites get a chance + // Resolve and run each test, continuing past failures so all tests get a chance // to run. Config/resolution errors abort immediately since they indicate a broken setup. var testFailures []string - for _, suiteName := range suiteNames { - suiteConfig, err := resolveTestSuiteByName(cfg, suiteName) + for _, testName := range testNames { + testConfig, err := resolveTestByName(cfg, testName) if err != nil { return err } - if err := runTestSuite(env, suiteConfig, imageConfig, options); err != nil { - slog.Error("Test suite failed", - slog.String("suite", suiteName), + if err := runTest(env, testConfig, imageConfig, options); err != nil { + slog.Error("Test failed", + slog.String("test", testName), slog.Any("error", err), ) - testFailures = append(testFailures, suiteName) + testFailures = append(testFailures, testName) } } if len(testFailures) > 0 { - return fmt.Errorf("%d of %d test suite(s) failed: %s", - len(testFailures), len(suiteNames), strings.Join(testFailures, ", ")) + return fmt.Errorf("%d of %d test(s) failed: %s", + len(testFailures), len(testNames), strings.Join(testFailures, ", ")) } return nil } -// resolveTestSuiteNames determines which test suites to run. If explicit names are -// provided, they are used as-is. Otherwise, all test suites associated with the image -// are returned. -func resolveTestSuiteNames( - imageConfig *projectconfig.ImageConfig, explicitSuites []string, +// expandImageTestRefs returns the deduplicated set of test names associated with +// an image — combining direct test refs and the membership of every referenced +// test-group. Group refs that do not resolve in the project config are silently +// skipped (already rejected by project validation; this is a defensive guard). +// Ordering is preserved (direct refs first, then group members in group order). +func expandImageTestRefs( + imageConfig *projectconfig.ImageConfig, cfg *projectconfig.ProjectConfig, ) []string { - if len(explicitSuites) > 0 { - return explicitSuites + seen := make(map[string]struct{}) + result := make([]string, 0) + + add := func(name string) { + if _, ok := seen[name]; ok { + return + } + + seen[name] = struct{}{} + result = append(result, name) } - return imageConfig.TestNames() + for _, name := range imageConfig.TestRefNames() { + add(name) + } + + for _, groupName := range imageConfig.TestRefGroups() { + group, ok := cfg.TestGroups[groupName] + if !ok { + continue + } + + for _, member := range group.Tests { + add(member) + } + } + + return result } -// warnUnassociatedSuites logs a warning for each explicitly requested test suite -// that is not referenced by the image's test configuration. -func warnUnassociatedSuites( - imageName string, imageConfig *projectconfig.ImageConfig, explicitSuites []string, -) { - imageTestNames := imageConfig.TestNames() +// resolveTestNames determines which tests to run. If explicit names are provided, +// they are used as-is (so users can deliberately invoke a test that is not yet +// associated with the image — useful for one-off debugging). Otherwise, all tests +// associated with the image are returned. +func resolveTestNames(imageTestNames, explicitTests []string) []string { + if len(explicitTests) > 0 { + return explicitTests + } + + return imageTestNames +} - for _, name := range explicitSuites { +// warnUnassociatedTests logs a warning for each explicitly requested test that is +// not part of the image's expanded test set (direct refs + group members). +func warnUnassociatedTests(imageName string, imageTestNames, explicitTests []string) { + for _, name := range explicitTests { if !slices.Contains(imageTestNames, name) { - slog.Warn("Test suite is not associated with image", - slog.String("suite", name), + slog.Warn("Test is not associated with image", + slog.String("test", name), slog.String("image", imageName), ) } } } -// resolveTestSuiteByName looks up a test suite by name in the project configuration. -func resolveTestSuiteByName( - cfg *projectconfig.ProjectConfig, suiteName string, -) (*projectconfig.TestSuiteConfig, error) { - suiteConfig, ok := cfg.TestSuites[suiteName] +// resolveTestByName looks up a test by name in the project configuration. +func resolveTestByName( + cfg *projectconfig.ProjectConfig, testName string, +) (*projectconfig.TestConfig, error) { + testConfig, ok := cfg.Tests[testName] if !ok { - availableSuites := lo.Keys(cfg.TestSuites) - sort.Strings(availableSuites) + availableTests := lo.Keys(cfg.Tests) + sort.Strings(availableTests) - if len(availableSuites) == 0 { + if len(availableTests) == 0 { return nil, fmt.Errorf( - "test suite %#q not found; no test suites defined in project configuration", suiteName) + "test %#q not found; no tests defined in project configuration", testName) } return nil, fmt.Errorf( - "test suite %#q not found; available test suites: %s", - suiteName, strings.Join(availableSuites, ", "), + "test %#q not found; available tests: %s", + testName, strings.Join(availableTests, ", "), ) } - return &suiteConfig, nil + return &testConfig, nil } -// runTestSuite dispatches a single test suite to the appropriate runner. -func runTestSuite( - env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, +// runTest dispatches a single test to the appropriate runner. +func runTest( + env *azldev.Env, testConfig *projectconfig.TestConfig, imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, ) error { - switch suiteConfig.Type { + switch testConfig.Type { case projectconfig.TestTypePytest: - return RunPytestSuite(env, suiteConfig, imageConfig, options) + return RunPytest(env, testConfig, imageConfig, options) case projectconfig.TestTypeLisa: - return fmt.Errorf("LISA test suites cannot be run locally via 'azldev image test'; "+ - "test suite %#q must be run through the LISA infrastructure", suiteConfig.Name) + return fmt.Errorf("LISA tests cannot be run locally via 'azldev image test'; "+ + "test %#q must be run through the LISA infrastructure", testConfig.Name) default: - return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) + return fmt.Errorf("unsupported test type %#q for test %#q", testConfig.Type, testConfig.Name) } } diff --git a/internal/app/azldev/cmds/image/test_test.go b/internal/app/azldev/cmds/image/test_test.go index 26ab6e2b..92474e8e 100644 --- a/internal/app/azldev/cmds/image/test_test.go +++ b/internal/app/azldev/cmds/image/test_test.go @@ -21,7 +21,7 @@ func TestNewImageTestCmd(t *testing.T) { func TestNewImageTestCmd_Flags(t *testing.T) { cmd := image.NewImageTestCmd() - assert.NotNil(t, cmd.Flags().Lookup("test-suite")) + assert.NotNil(t, cmd.Flags().Lookup("test")) assert.NotNil(t, cmd.Flags().Lookup("image-path")) assert.NotNil(t, cmd.Flags().Lookup("junit-xml")) } diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index b481b35b..0ec591f8 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -303,6 +303,32 @@ type ComponentConfig struct { // all packages produced by this component. Overridden by package-group and per-package settings // for binary and debuginfo channels. Publish ComponentPublishConfig `toml:"publish,omitempty" json:"publish,omitempty" table:"-" jsonschema:"title=Publish settings,description=Component-level publish channel settings" fingerprint:"-"` + + // Tests holds the test configuration for this component, including which tests + // and test-groups apply to it. + Tests ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Test configuration for this component" fingerprint:"-"` +} + +// ComponentTestsConfig holds the test-related configuration for a component. +// It mirrors [ImageTestsConfig] in shape: a list of [TestRef] entries that +// each name a test or a test-group. +type ComponentTestsConfig struct { + // Tests is the list of test references that apply to this component. Each entry + // must reference either a single test (by name, key in [tests]) or a + // test-group (by group, key in [test-groups]). + Tests []TestRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this component"` +} + +// TestRefNames returns the names of [TestConfig]s directly referenced by this component +// (i.e., entries with [TestRef.Name] set). Group references are excluded. +func (c *ComponentConfig) TestRefNames() []string { + return testRefNames(c.Tests.Tests) +} + +// TestRefGroups returns the names of [TestGroupConfig]s referenced by this component +// (i.e., entries with [TestRef.Group] set). Direct test references are excluded. +func (c *ComponentConfig) TestRefGroups() []string { + return testRefGroups(c.Tests.Tests) } // AllowedSourceFilesHashTypes defines the set of hash types that are supported @@ -400,6 +426,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi // absolutized; nothing reads it afterward, so preserve the original value verbatim // rather than redundantly re-rooting each glob. OverlayFiles: slices.Clone(c.OverlayFiles), + Tests: deep.MustCopy(c.Tests), } // Fix up paths. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e16b79b7..3120e8b6 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -63,8 +63,13 @@ type ConfigFile struct { // to be applied to sets of binary packages. PackageGroups map[string]PackageGroupConfig `toml:"package-groups,omitempty" validate:"dive" jsonschema:"title=Package groups,description=Definitions of package groups for shared binary package configuration"` - // Definitions of test suites. - TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Test Suites,description=Definitions of test suites for this project"` + // Definitions of tests. A test is a single unit of testing: one configuration + // of one runner / harness (e.g. pytest) with framework-specific options. + Tests map[string]TestConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Tests,description=Definitions of tests for this project"` + + // Definitions of test-groups. A test-group is a named bundle of tests, referenced + // from images and components via [TestRef] entries with a 'group' field. + TestGroups map[string]TestGroupConfig `toml:"test-groups,omitempty" validate:"dive" jsonschema:"title=Test Groups,description=Definitions of test-groups (named bundles of tests) for this project"` // Internal fields used to track the origin of the config file; `dir` is the directory // that the config file's relative paths are based from. @@ -131,19 +136,50 @@ func (f ConfigFile) Validate() error { } } - // Validate test suite configurations. - for suiteName, suite := range f.TestSuites { - // Suite names are used as path components (e.g., for the per-suite venv directory), - // so reject anything that could escape the intended directory or otherwise be unsafe - // across platforms. - if err := fileutils.ValidateFilename(suiteName); err != nil { - return fmt.Errorf("invalid test suite name %#q:\n%w", suiteName, err) + // Validate test configurations. + for testName, test := range f.Tests { + // Test names are used as path components (e.g., for a per-test venv directory), + // so reject anything that could escape the intended directory or otherwise be + // unsafe across platforms. + if err := fileutils.ValidateFilename(testName); err != nil { + return fmt.Errorf("invalid test name %#q:\n%w", testName, err) } - suite.Name = suiteName + test.Name = testName - if err := suite.Validate(); err != nil { - return fmt.Errorf("invalid test suite %#q:\n%w", suiteName, err) + if err := test.Validate(); err != nil { + return fmt.Errorf("invalid test %#q:\n%w", testName, err) + } + } + + // Validate test-group configurations. + for groupName, group := range f.TestGroups { + if err := fileutils.ValidateFilename(groupName); err != nil { + return fmt.Errorf("invalid test-group name %#q:\n%w", groupName, err) + } + + group.Name = groupName + + if err := group.Validate(); err != nil { + return fmt.Errorf("invalid test-group %#q:\n%w", groupName, err) + } + } + + // Validate that test refs on each image have well-formed shape (exactly one of + // name/group). Cross-resolution against the top-level tests/test-groups maps is + // performed at project level. + for imageName, image := range f.Images { + ctx := fmt.Sprintf("image %#q tests.tests", imageName) + if err := validateTestRefs(ctx, image.Tests.Tests); err != nil { + return err + } + } + + // Same for components. + for componentName, component := range f.Components { + ctx := fmt.Sprintf("component %#q tests.tests", componentName) + if err := validateTestRefs(ctx, component.Tests.Tests); err != nil { + return err } } diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index d61f2866..71e11162 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -12,6 +12,84 @@ import ( "github.com/stretchr/testify/require" ) +func TestProjectConfigFileValidation_TestMismatchedSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestConfig{ + "smoke": { + Type: projectconfig.TestTypeLisa, + Pytest: &projectconfig.PytestConfig{WorkingDir: "tests"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "invalid test") + assert.Contains(t, err.Error(), "smoke") + assert.Contains(t, err.Error(), "pytest") +} + +func TestProjectConfigFileValidation_TestMatchingSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestConfig{ + "smoke": { + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{WorkingDir: "tests"}, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_TestMissingTypeField(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestConfig{ + "smoke": { + Pytest: &projectconfig.PytestConfig{WorkingDir: "tests"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "type") +} + +func TestProjectConfigFileValidation_TestRefMissingNameAndGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + Images: map[string]projectconfig.ImageConfig{ + "vm-base": { + Tests: projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{{}}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidTestRef) +} + +func TestProjectConfigFileValidation_TestRefBothNameAndGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "kernel": { + Tests: projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidTestRef) +} + func TestProjectConfigFileValidation_EmptyFile(t *testing.T) { file := projectconfig.ConfigFile{} assert.NoError(t, file.Validate()) diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 9fe56870..151764ea 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -65,6 +65,11 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // ComponentConfig.Publish — post-build routing (where to publish), not a build input. "ComponentConfig.Publish": true, + // ComponentConfig.Tests — test associations are metadata about which tests apply + // to this component, not inputs to the build. They must not drift lock fingerprints + // — adding/removing a test ref should never trigger a rebuild. + "ComponentConfig.Tests": true, + // ComponentOverlay.Description — human-readable documentation for the overlay. "ComponentOverlay.Description": true, // ComponentOverlay.Source — absolute path that varies by checkout location. diff --git a/internal/projectconfig/image.go b/internal/projectconfig/image.go index f2000851..18757274 100644 --- a/internal/projectconfig/image.go +++ b/internal/projectconfig/image.go @@ -28,8 +28,8 @@ type ImageConfig struct { // Capabilities describes the features and properties of this image. Capabilities ImageCapabilities `toml:"capabilities,omitempty" json:"capabilities,omitempty" jsonschema:"title=Capabilities,description=Features and properties of this image"` - // Tests holds the test configuration for this image, including which test suites - // apply to it. + // Tests holds the test configuration for this image, including which tests + // and test-groups apply to it. Tests ImageTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Test configuration for this image"` // Publish holds the publish settings for this image. @@ -111,27 +111,48 @@ func (c *ImageCapabilities) EnabledNames() []string { // ImageTestsConfig holds the test-related configuration for an image. type ImageTestsConfig struct { - // TestSuites is the list of test suite references that apply to this image. Each - // reference identifies a test suite defined in the top-level [test-suites] section - // and may carry per-test metadata in the future (e.g., required vs optional). - TestSuites []TestSuiteRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=List of test suite references that apply to this image"` + // Tests is the list of test references that apply to this image. Each entry + // must reference either a single test (by name, key in [tests]) or a + // test-group (by group, key in [test-groups]). + Tests []TestRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this image"` } -// TestSuiteRef is a reference to a named test suite. Using a structured type (rather than -// a bare string) allows per-test metadata to be added later without a breaking config change. -type TestSuiteRef struct { - // Name is the key into the top-level [test-suites] map. - Name string `toml:"name" json:"name" jsonschema:"required,title=Name,description=Name of the test suite (must match a key in [test-suites])"` +// TestRefNames returns the names of [TestConfig]s directly referenced by this image +// (i.e., entries with [TestRef.Name] set). Group references are excluded. +func (i *ImageConfig) TestRefNames() []string { + return testRefNames(i.Tests.Tests) } -// TestNames returns the test suite names referenced by this image. -func (i *ImageConfig) TestNames() []string { - names := make([]string, len(i.Tests.TestSuites)) - for idx, ref := range i.Tests.TestSuites { - names[idx] = ref.Name +// TestRefGroups returns the names of [TestGroupConfig]s referenced by this image +// (i.e., entries with [TestRef.Group] set). Direct test references are excluded. +func (i *ImageConfig) TestRefGroups() []string { + return testRefGroups(i.Tests.Tests) +} + +// testRefNames extracts the Name field from each [TestRef] that has one set. +func testRefNames(refs []TestRef) []string { + out := make([]string, 0, len(refs)) + + for _, ref := range refs { + if ref.Name != "" { + out = append(out, ref.Name) + } } - return names + return out +} + +// testRefGroups extracts the Group field from each [TestRef] that has one set. +func testRefGroups(refs []TestRef) []string { + out := make([]string, 0, len(refs)) + + for _, ref := range refs { + if ref.Group != "" { + out = append(out, ref.Group) + } + } + + return out } // Defines where to find an image definition. diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index ab6227b5..ea4682b5 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -44,7 +44,8 @@ func loadAndResolveProjectConfig( Distros: make(map[string]DistroDefinition), GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), - TestSuites: make(map[string]TestSuiteConfig), + Tests: make(map[string]TestConfig), + TestGroups: make(map[string]TestGroupConfig), } for _, configFilePath := range configFilePaths { @@ -142,7 +143,11 @@ func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return err } - if err := mergeTestSuites(resolvedCfg, loadedCfg); err != nil { + if err := mergeTests(resolvedCfg, loadedCfg); err != nil { + return err + } + + if err := mergeTestGroups(resolvedCfg, loadedCfg); err != nil { return err } @@ -301,19 +306,37 @@ func mergePackageGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error return nil } -// mergeTestSuites merges test suite definitions from a loaded config file into the -// resolved config. Duplicate test suite names are not allowed. -func mergeTestSuites(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { - for suiteName, suite := range loadedCfg.TestSuites { - if _, ok := resolvedCfg.TestSuites[suiteName]; ok { - return fmt.Errorf("%w: test suite %#q", ErrDuplicateTestSuites, suiteName) +// mergeTests merges test definitions from a loaded config file into the +// resolved config. Duplicate test names are not allowed. +func mergeTests(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { + for testName, test := range loadedCfg.Tests { + if _, ok := resolvedCfg.Tests[testName]; ok { + return fmt.Errorf("%w: test %#q", ErrDuplicateTests, testName) + } + + // Fill out fields not explicitly serialized. + test.Name = testName + test.SourceConfigFile = loadedCfg + + resolvedCfg.Tests[testName] = *(test.WithAbsolutePaths(loadedCfg.dir)) + } + + return nil +} + +// mergeTestGroups merges test-group definitions from a loaded config file into the +// resolved config. Duplicate test-group names are not allowed. +func mergeTestGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { + for groupName, group := range loadedCfg.TestGroups { + if _, ok := resolvedCfg.TestGroups[groupName]; ok { + return fmt.Errorf("%w: test-group %#q", ErrDuplicateTestGroups, groupName) } // Fill out fields not explicitly serialized. - suite.Name = suiteName - suite.SourceConfigFile = loadedCfg + group.Name = groupName + group.SourceConfigFile = loadedCfg - resolvedCfg.TestSuites[suiteName] = *(suite.WithAbsolutePaths(loadedCfg.dir)) + resolvedCfg.TestGroups[groupName] = *(group.WithAbsolutePaths(loadedCfg.dir)) } return nil diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index f825d55b..d99dc871 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -827,7 +827,7 @@ rpm-channel = "devel" } } -func TestLoadAndResolveProjectConfig_TestSuite(t *testing.T) { +func TestLoadAndResolveProjectConfig_Test(t *testing.T) { const configContents = ` [test-suites.smoke] type = "pytest" @@ -847,11 +847,11 @@ extra-args = ["--image-path", "{image-path}"] config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.NoError(t, err) - require.Len(t, config.TestSuites, 1) + require.Len(t, config.Tests, 1) // Check pytest test. - if assert.Contains(t, config.TestSuites, "smoke") { - smokeTest := config.TestSuites["smoke"] + if assert.Contains(t, config.Tests, "smoke") { + smokeTest := config.Tests["smoke"] assert.Equal(t, "smoke", smokeTest.Name) assert.Equal(t, TestTypePytest, smokeTest.Type) assert.Equal(t, "Smoke tests for images", smokeTest.Description) @@ -895,7 +895,7 @@ test-paths = ["other/"] } _, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) - require.ErrorIs(t, err, ErrDuplicateTestSuites) + require.ErrorIs(t, err, ErrDuplicateTests) } func TestLoadAndResolveProjectConfig_InvalidTestType(t *testing.T) { @@ -927,7 +927,7 @@ type = "pytest" assert.ErrorIs(t, err, ErrMissingTestField) } -func TestLoadAndResolveProjectConfig_TestSuiteMissingType(t *testing.T) { +func TestLoadAndResolveProjectConfig_TestMissingType(t *testing.T) { const configContents = ` [test-suites.smoke] # 'type' intentionally omitted. @@ -943,7 +943,7 @@ description = "no type set" assert.Contains(t, err.Error(), "type") } -func TestLoadAndResolveProjectConfig_TestSuiteInvalidName(t *testing.T) { +func TestLoadAndResolveProjectConfig_TestInvalidName(t *testing.T) { // Names containing path separators or traversal segments must be rejected at // config load time since they are used as path components (e.g., venv directories). const configContents = ` @@ -959,7 +959,7 @@ working-dir = "tests" _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.Error(t, err) - assert.Contains(t, err.Error(), "invalid test suite name") + assert.Contains(t, err.Error(), "invalid test name") } func TestLoadAndResolveProjectConfig_ImageWithValidTestRef(t *testing.T) { @@ -985,7 +985,7 @@ test-suites = [{ name = "smoke" }] require.NoError(t, err) if assert.Contains(t, config.Images, "myimage") { - assert.Equal(t, []TestSuiteRef{{Name: "smoke"}}, config.Images["myimage"].Tests.TestSuites) + assert.Equal(t, []TestRef{{Name: "smoke"}}, config.Images["myimage"].Tests.Tests) } } @@ -1003,7 +1003,7 @@ test-suites = [{ name = "nonexistent" }] _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.Error(t, err) - require.ErrorIs(t, err, ErrUndefinedTestSuite) + require.ErrorIs(t, err, ErrUndefinedTest) assert.Contains(t, err.Error(), "nonexistent") } @@ -1105,7 +1105,7 @@ rpm-channel = "new-channel" "rpm-channel should take precedence over the deprecated channel field") } -func TestLoadAndResolveProjectConfig_TestSuiteInstallMode(t *testing.T) { +func TestLoadAndResolveProjectConfig_TestInstallMode(t *testing.T) { const configContents = ` [test-suites.smoke] type = "pytest" @@ -1122,15 +1122,15 @@ test-paths = ["cases/"] config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.NoError(t, err) - if assert.Contains(t, config.TestSuites, "smoke") { - smokeTest := config.TestSuites["smoke"] + if assert.Contains(t, config.Tests, "smoke") { + smokeTest := config.Tests["smoke"] require.NotNil(t, smokeTest.Pytest) assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.Install) assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.EffectiveInstallMode()) } } -func TestLoadAndResolveProjectConfig_TestSuiteInvalidInstallMode(t *testing.T) { +func TestLoadAndResolveProjectConfig_TestInvalidInstallMode(t *testing.T) { const configContents = ` [test-suites.smoke] type = "pytest" diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index adb862aa..32ac5fe7 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -44,8 +44,11 @@ type ProjectConfig struct { // Definitions of package groups with shared configuration. PackageGroups map[string]PackageGroupConfig `toml:"package-groups,omitempty" json:"packageGroups,omitempty" jsonschema:"title=Package groups,description=Mapping of package group names to configurations for publish-time routing"` - // Definitions of test suites. - TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=Mapping of test suite names to configurations"` + // Definitions of tests. + Tests map[string]TestConfig `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Tests,description=Mapping of test names to configurations"` + + // Definitions of test-groups. + TestGroups map[string]TestGroupConfig `toml:"test-groups,omitempty" json:"testGroups,omitempty" jsonschema:"title=Test Groups,description=Mapping of test-group names to configurations"` // Root config file path; not serialized. RootConfigFilePath string `toml:"-" json:"-"` @@ -64,7 +67,8 @@ func NewProjectConfig() ProjectConfig { Resources: ResourcesConfig{RpmRepos: make(map[string]RpmRepoResource)}, GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), - TestSuites: make(map[string]TestSuiteConfig), + Tests: make(map[string]TestConfig), + TestGroups: make(map[string]TestGroupConfig), } } @@ -83,7 +87,15 @@ func (cfg *ProjectConfig) Validate() error { return err } - if err := validateImageTestReferences(cfg.Images, cfg.TestSuites); err != nil { + if err := validateTestGroupMembership(cfg.TestGroups, cfg.Tests); err != nil { + return err + } + + if err := validateImageTestReferences(cfg.Images, cfg.Tests, cfg.TestGroups); err != nil { + return err + } + + if err := validateComponentTestReferences(cfg.Components, cfg.Tests, cfg.TestGroups); err != nil { return err } @@ -268,22 +280,118 @@ func validatePackageGroupMembership(groups map[string]PackageGroupConfig) error return nil } -// validateImageTestReferences checks that every test suite name in an image's -// [ImageConfig.Tests.TestSuites] list corresponds to a defined entry in the top-level -// TestSuites map. -func validateImageTestReferences(images map[string]ImageConfig, testSuites map[string]TestSuiteConfig) error { - for imageName, image := range images { - for _, suiteName := range image.TestNames() { - if _, ok := testSuites[suiteName]; !ok { - return fmt.Errorf( - "%w: image %#q references test suite %#q, which is not defined in [test-suites]", - ErrUndefinedTestSuite, imageName, suiteName, - ) +// validateImageTestReferences checks that every test name in an image's +// [ImageConfig.Tests.Tests] list resolves to a defined entry in the top-level +// [ProjectConfig.Tests] map, and that every group name resolves to an entry in +// [ProjectConfig.TestGroups]. All offending references are reported together. +func validateImageTestReferences( + images map[string]ImageConfig, + tests map[string]TestConfig, + testGroups map[string]TestGroupConfig, +) error { + var errs []error + + // Iterate in sorted order for deterministic error reporting. + imageNames := make([]string, 0, len(images)) + for name := range images { + imageNames = append(imageNames, name) + } + + sort.Strings(imageNames) + + for _, imageName := range imageNames { + image := images[imageName] + for _, testName := range image.TestRefNames() { + if _, ok := tests[testName]; !ok { + errs = append(errs, fmt.Errorf( + "%w: image %#q references test %#q, which is not defined in [tests]", + ErrUndefinedTest, imageName, testName, + )) + } + } + + for _, groupName := range image.TestRefGroups() { + if _, ok := testGroups[groupName]; !ok { + errs = append(errs, fmt.Errorf( + "%w: image %#q references test-group %#q, which is not defined in [test-groups]", + ErrUndefinedTestGroup, imageName, groupName, + )) } } } - return nil + return errors.Join(errs...) +} + +// validateComponentTestReferences mirrors [validateImageTestReferences] for +// component-side test associations. +func validateComponentTestReferences( + components map[string]ComponentConfig, + tests map[string]TestConfig, + testGroups map[string]TestGroupConfig, +) error { + var errs []error + + componentNames := make([]string, 0, len(components)) + for name := range components { + componentNames = append(componentNames, name) + } + + sort.Strings(componentNames) + + for _, componentName := range componentNames { + component := components[componentName] + for _, testName := range component.TestRefNames() { + if _, ok := tests[testName]; !ok { + errs = append(errs, fmt.Errorf( + "%w: component %#q references test %#q, which is not defined in [tests]", + ErrUndefinedTest, componentName, testName, + )) + } + } + + for _, groupName := range component.TestRefGroups() { + if _, ok := testGroups[groupName]; !ok { + errs = append(errs, fmt.Errorf( + "%w: component %#q references test-group %#q, which is not defined in [test-groups]", + ErrUndefinedTestGroup, componentName, groupName, + )) + } + } + } + + return errors.Join(errs...) +} + +// validateTestGroupMembership checks that every test name listed in each +// [TestGroupConfig.Tests] member list resolves to a defined entry in the +// top-level [ProjectConfig.Tests] map. All offending references are reported +// together. Nested group references are not supported and never resolved. +func validateTestGroupMembership( + testGroups map[string]TestGroupConfig, tests map[string]TestConfig, +) error { + var errs []error + + groupNames := make([]string, 0, len(testGroups)) + for name := range testGroups { + groupNames = append(groupNames, name) + } + + sort.Strings(groupNames) + + for _, groupName := range groupNames { + group := testGroups[groupName] + for _, member := range group.Tests { + if _, ok := tests[member]; !ok { + errs = append(errs, fmt.Errorf( + "%w: test-group %#q references test %#q, which is not defined in [tests]", + ErrUndefinedTest, groupName, member, + )) + } + } + } + + return errors.Join(errs...) } // Default project-relative paths used when the corresponding [ProjectInfo] diff --git a/internal/projectconfig/test.go b/internal/projectconfig/test.go new file mode 100644 index 00000000..79a19a87 --- /dev/null +++ b/internal/projectconfig/test.go @@ -0,0 +1,419 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "slices" + + "dario.cat/mergo" + "github.com/invopop/jsonschema" + orderedmap "github.com/pb33f/ordered-map/v2" +) + +// TestType indicates the type of test framework used to run a test. +type TestType string + +const ( + // TestTypePytest uses pytest to run static/offline validation checks. + TestTypePytest TestType = "pytest" + // TestTypeLisa uses LISA (Linux Integration Services Automation) to run + // VM-level tests. LISA tests are not executed by azldev directly; the type + // serves as metadata for external orchestration. + TestTypeLisa TestType = "lisa" +) + +// TestKind classifies the nature of a test for cost/expectation reporting. +// It is a closed multi-valued enum: a single test may legitimately combine +// kinds (e.g., a regression test that also gathers performance data). +type TestKind string + +const ( + // TestKindFunctional indicates a functional / regression test: it asserts + // behavior against an expected outcome. This is the typical default kind. + TestKindFunctional TestKind = "functional" + // TestKindPerformance indicates a test whose primary purpose is to measure + // performance characteristics (throughput, latency, etc.) rather than to + // assert pass/fail behavior. + TestKindPerformance TestKind = "performance" +) + +var ( + // ErrDuplicateTests is returned when duplicate conflicting test definitions are found. + ErrDuplicateTests = errors.New("duplicate test") + // ErrUnknownTestType is returned for unrecognized test types. + ErrUnknownTestType = errors.New("unknown test type") + // ErrUnknownTestKind is returned for unrecognized test kind values. + ErrUnknownTestKind = errors.New("unknown test kind") + // ErrMissingTestField is returned when a required test config field is missing. + ErrMissingTestField = errors.New("missing required test field") + // ErrUndefinedTest is returned when something references a test name that is not defined. + ErrUndefinedTest = errors.New("undefined test reference") + // ErrMismatchedTestSubtable is returned when a test config has a subtable + // that does not match its declared type. + ErrMismatchedTestSubtable = errors.New("mismatched test subtable") + // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. + ErrInvalidInstallMode = errors.New("invalid install mode") + // ErrInvalidTestRef is returned when a [TestRef] has neither or both of name/group set. + ErrInvalidTestRef = errors.New("invalid test reference") +) + +// TestConfig defines a single named unit of testing: one configuration of one +// runner / harness (e.g. pytest) with framework-specific options. Tests +// are referenced from images and components via [TestRef] entries, and may be +// bundled into [TestGroupConfig]s. +// +// JSONSchemaExtend uses a value receiver because the invopop/jsonschema library +// only invokes value-receiver methods when reflecting on the type; the rest of +// the methods use pointer receivers because they mutate. +// +//nolint:recvcheck // intentional mixed receivers (see comment above). +type TestConfig struct { + // The test's name; not present in serialized TOML files (populated from the map key). + Name string `toml:"-" json:"name" table:",sortkey"` + + // Description of the test. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test"` + + // Type indicates the test framework to use. + Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,enum=lisa,title=Type,description=Type of test framework (pytest or lisa)"` + + // Kind classifies the test's nature (functional, performance, ...). May be + // multi-valued for tests that legitimately combine kinds (e.g., a regression + // test that also gathers performance data). Optional; defaults to unspecified. + Kind []TestKind `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Classification of the test's nature (e.g. functional\\, performance). Closed enum; multi-valued."` + + // LongRunning marks a test that is expected to take a long time (hours). + // Consumers (e.g., CI orchestration) can use this to schedule appropriately. + // Optional; omitted when false. This is a declarative hint about cost, not + // a configurable timeout. + LongRunning bool `toml:"long-running,omitempty" json:"longRunning,omitempty" jsonschema:"title=Long running,description=Indicates that this test may take hours to complete. Hint about cost\\, not a configurable timeout."` + + // Pytest holds pytest-specific configuration. Required when Type is "pytest". + Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` + + // Reference to the source config file that this definition came from; not present + // in serialized files. + SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` +} + +// PytestInstallMode specifies how Python dependencies are installed for a pytest test. +type PytestInstallMode string + +const ( + // PytestInstallPyproject installs dependencies from pyproject.toml using editable mode. + // Returns an error if pyproject.toml is not found in the working directory. + PytestInstallPyproject PytestInstallMode = "pyproject" + // PytestInstallRequirements installs dependencies from requirements.txt. + // Returns an error if requirements.txt is not found. + PytestInstallRequirements PytestInstallMode = "requirements" + // PytestInstallNone skips dependency installation entirely. This is the default + // when [PytestConfig.Install] is not specified — pytest must already be available + // in the venv (e.g., pre-installed, or installed by the test author out-of-band). + PytestInstallNone PytestInstallMode = "none" +) + +// PytestConfig holds configuration specific to pytest-based tests. +type PytestConfig struct { + // WorkingDir is the directory to use as the current working directory when running pytest. + // Relative paths are resolved against the config file's directory. + WorkingDir string `toml:"working-dir,omitempty" json:"workingDir,omitempty" jsonschema:"title=Working directory,description=Directory to use as CWD when running pytest"` + + // TestPaths is the list of test file paths or directories to pass to pytest as positional + // arguments. Glob patterns (e.g., cases/test_*.py) are expanded relative to WorkingDir. + TestPaths []string `toml:"test-paths,omitempty" json:"testPaths,omitempty" jsonschema:"title=Test paths,description=Test file paths or directories passed to pytest. Glob patterns are expanded."` + + // ExtraArgs is the list of additional arguments to pass to pytest. These are passed + // verbatim after placeholder substitution. Use {image-path} as a placeholder for the + // image path, which will be substituted at runtime. + ExtraArgs []string `toml:"extra-args,omitempty" json:"extraArgs,omitempty" jsonschema:"title=Extra arguments,description=Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path."` + + // Install specifies how Python dependencies are installed into the venv before running + // pytest. Defaults to "none" (no install) when not specified. + Install PytestInstallMode `toml:"install,omitempty" json:"install,omitempty" jsonschema:"enum=pyproject,enum=requirements,enum=none,title=Install mode,description=How to install Python dependencies: pyproject\\, requirements\\, or none (default)"` +} + +// Validate checks that the test config has valid type-specific required fields and that +// only the matching subtable is present. +func (t *TestConfig) Validate() error { + if t.Type == "" { + return fmt.Errorf("%w: test %#q is missing required field 'type'", + ErrMissingTestField, t.Name) + } + + if err := validateTestKinds(t.Name, t.Kind); err != nil { + return err + } + + switch t.Type { + case TestTypePytest: + if t.Pytest == nil { + return fmt.Errorf("%w: test %#q of type %#q requires a [pytest] subtable", + ErrMissingTestField, t.Name, t.Type) + } + + if err := t.Pytest.Validate(); err != nil { + return fmt.Errorf("test %#q: %w", t.Name, err) + } + + case TestTypeLisa: + // LISA is an external test framework not executed by azldev. + // Tests of this type serve as metadata for external orchestration (e.g. control tower). + if t.Pytest != nil { + return fmt.Errorf( + "%w: test %#q of type %#q cannot include subtable 'pytest'", + ErrMismatchedTestSubtable, t.Name, t.Type, + ) + } + + default: + return fmt.Errorf("%w: %#q (test: %#q)", ErrUnknownTestType, t.Type, t.Name) + } + + return nil +} + +// validateTestKinds checks that every kind value is a recognized [TestKind] and +// flags duplicates (kept order-preserving by reporting the first dup encountered). +func validateTestKinds(testName string, kinds []TestKind) error { + seen := make(map[TestKind]bool, len(kinds)) + + for _, k := range kinds { + if !k.isValid() { + return fmt.Errorf("%w: %#q (test: %#q); allowed values: %#q, %#q", + ErrUnknownTestKind, k, testName, TestKindFunctional, TestKindPerformance) + } + + if seen[k] { + return fmt.Errorf("test %#q has duplicate kind %#q", testName, k) + } + + seen[k] = true + } + + return nil +} + +// isValid returns whether the kind is a recognized [TestKind] value. +func (k TestKind) isValid() bool { + switch k { + case TestKindFunctional, TestKindPerformance: + return true + default: + return false + } +} + +// Validate checks that the [PytestConfig] fields are valid. +func (p *PytestConfig) Validate() error { + if p.Install != "" && !p.Install.isValid() { + return fmt.Errorf( + "%w: %#q; allowed values: %#q, %#q, %#q (or omit for default %#q)", + ErrInvalidInstallMode, p.Install, + PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone, + PytestInstallNone, + ) + } + + // When the effective install mode requires a working directory, 'working-dir' + // must be specified. The default mode is 'none' (no install) and so requires + // nothing; only an explicitly-set install mode that performs work needs the dir. + if p.EffectiveInstallMode() != PytestInstallNone && p.WorkingDir == "" { + return fmt.Errorf( + "%w: 'working-dir' is required when install mode is %#q", + ErrMissingTestField, p.EffectiveInstallMode(), + ) + } + + return nil +} + +// EffectiveInstallMode returns the install mode, defaulting to [PytestInstallNone] when +// the field is not set. +func (p *PytestConfig) EffectiveInstallMode() PytestInstallMode { + if p.Install == "" { + return PytestInstallNone + } + + return p.Install +} + +// isValid returns whether the mode is a recognized [PytestInstallMode] value. +func (m PytestInstallMode) isValid() bool { + switch m { + case PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone: + return true + default: + return false + } +} + +// MergeUpdatesFrom updates the test config with overrides present in other. +func (t *TestConfig) MergeUpdatesFrom(other *TestConfig) error { + err := mergo.Merge(t, other, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + return fmt.Errorf("failed to merge test config:\n%w", err) + } + + return nil +} + +// WithAbsolutePaths returns a copy of the test config with relative file paths converted +// to absolute paths (relative to referenceDir). +func (t *TestConfig) WithAbsolutePaths(referenceDir string) *TestConfig { + result := &TestConfig{ + Name: t.Name, + Description: t.Description, + Type: t.Type, + Kind: slices.Clone(t.Kind), + LongRunning: t.LongRunning, + SourceConfigFile: t.SourceConfigFile, + } + + if t.Pytest != nil { + result.Pytest = &PytestConfig{ + WorkingDir: makeAbsolute(referenceDir, t.Pytest.WorkingDir), + TestPaths: append([]string(nil), t.Pytest.TestPaths...), + ExtraArgs: append([]string(nil), t.Pytest.ExtraArgs...), + Install: t.Pytest.Install, + } + } + + return result +} + +// TestRef is a reference from an image or component to either a single named test +// or a named test-group. Exactly one of [TestRef.Name] / [TestRef.Group] must be set. +// +// Using a structured ref (rather than a bare string) lets the same association +// list ergonomically mix tests and groups, and leaves room for per-ref metadata +// (e.g., "required vs optional") to be added without a breaking config change. +// +// JSONSchemaExtend uses a value receiver because the invopop/jsonschema library +// only invokes value-receiver methods when reflecting on the type; [TestRef.Validate] +// uses a pointer receiver for consistency with the rest of this package. +// +//nolint:recvcheck // intentional mixed receivers (see comment above). +type TestRef struct { + // Name references a single [TestConfig] by key. + Name string `toml:"name,omitempty" json:"name,omitempty" jsonschema:"title=Name,description=Name of a test (key in [tests]); mutually exclusive with 'group'"` + + // Group references a [TestGroupConfig] by key. + Group string `toml:"group,omitempty" json:"group,omitempty" jsonschema:"title=Group,description=Name of a test-group (key in [test-groups]); mutually exclusive with 'name'"` +} + +// Validate checks that exactly one of Name/Group is set on the [TestRef]. +func (r *TestRef) Validate(context string) error { + if r.Name == "" && r.Group == "" { + return fmt.Errorf("%w: %s must set either 'name' or 'group'", + ErrInvalidTestRef, context) + } + + if r.Name != "" && r.Group != "" { + return fmt.Errorf( + "%w: %s sets both 'name' = %#q and 'group' = %#q; exactly one is allowed", + ErrInvalidTestRef, context, r.Name, r.Group, + ) + } + + return nil +} + +// validateTestRefs runs [TestRef.Validate] on every entry and joins errors. +func validateTestRefs(context string, refs []TestRef) error { + var errs []error + + for i, ref := range refs { + entryContext := fmt.Sprintf("%s entry %d", context, i) + if err := ref.Validate(entryContext); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +// orderedMapWithConst returns an ordered map containing a single property whose +// schema is `{ "const": value }`. Helper for building `If` schemas that match a +// specific discriminator value. +func orderedMapWithConst(key string, value any) *orderedmap.OrderedMap[string, *jsonschema.Schema] { + m := orderedmap.New[string, *jsonschema.Schema]() + m.Set(key, &jsonschema.Schema{Const: value}) + + return m +} + +// JSONSchemaExtend tightens the generated schema for [TestRef] so editors can +// flag refs that set neither or both of name/group, or empty/whitespace-only +// values. Mirrors the runtime constraints in [TestRef.Validate]; the runtime +// validator is the source of truth. +func (TestRef) JSONSchemaExtend(schema *jsonschema.Schema) { + // Exactly one of name|group. Encoded as oneOf with `required` on each and + // `not` excluding the other, which forbids both `{}` and `{name, group}`. + schema.OneOf = []*jsonschema.Schema{ + {Required: []string{"name"}, Not: &jsonschema.Schema{Required: []string{"group"}}}, + {Required: []string{"group"}, Not: &jsonschema.Schema{Required: []string{"name"}}}, + } + + // Reject empty / leading-whitespace identifiers in both name and group. + minLen := uint64(1) + + if schema.Properties != nil { + if nameProp, ok := schema.Properties.Get("name"); ok && nameProp != nil { + nameProp.MinLength = &minLen + nameProp.Pattern = `^\S` + } + + if groupProp, ok := schema.Properties.Get("group"); ok && groupProp != nil { + groupProp.MinLength = &minLen + groupProp.Pattern = `^\S` + } + } +} + +// JSONSchemaExtend tightens [TestConfig] so the framework-specific subtable +// must match the declared `type`, and so the `kind` field is a closed, +// duplicate-free enum. Mirrors the runtime checks in [TestConfig.Validate] +// (subtable/type match) and [validateTestKinds] (closed enum + no duplicates); +// the runtime validator is the source of truth. +// +// - type=pytest requires `pytest`; +// - type=lisa forbids `pytest` (no subtable today). +func (TestConfig) JSONSchemaExtend(schema *jsonschema.Schema) { + // Constrain `kind` items to the closed [TestKind] enum and forbid duplicates, + // mirroring validateTestKinds. The struct tag only sets the array's title and + // description; the item-level enum has to be applied here. + if schema.Properties != nil { + if kindProp, ok := schema.Properties.Get("kind"); ok && kindProp != nil { + kindProp.UniqueItems = true + + if kindProp.Items != nil { + kindProp.Items.Enum = []any{ + string(TestKindFunctional), + string(TestKindPerformance), + } + } + } + } + + onlyPytest := &jsonschema.Schema{ + Required: []string{"pytest"}, + } + + noSubtable := &jsonschema.Schema{ + Not: &jsonschema.Schema{Required: []string{"pytest"}}, + } + + schema.AllOf = append(schema.AllOf, + &jsonschema.Schema{ + If: &jsonschema.Schema{Properties: orderedMapWithConst("type", string(TestTypePytest))}, + Then: onlyPytest, + }, + &jsonschema.Schema{ + If: &jsonschema.Schema{Properties: orderedMapWithConst("type", string(TestTypeLisa))}, + Then: noSubtable, + }, + ) +} diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/test_test.go similarity index 78% rename from internal/projectconfig/testsuite_test.go rename to internal/projectconfig/test_test.go index e24f758f..cbd6b91d 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/test_test.go @@ -47,28 +47,28 @@ func TestImageCapabilities_EnabledNames(t *testing.T) { }) } -func TestImageConfig_TestNames(t *testing.T) { +func TestImageConfig_TestRefNames(t *testing.T) { t.Run("with tests", func(t *testing.T) { img := projectconfig.ImageConfig{ Tests: projectconfig.ImageTestsConfig{ - TestSuites: []projectconfig.TestSuiteRef{ + Tests: []projectconfig.TestRef{ {Name: "smoke"}, {Name: "integration"}, }, }, } - assert.Equal(t, []string{"smoke", "integration"}, img.TestNames()) + assert.Equal(t, []string{"smoke", "integration"}, img.TestRefNames()) }) t.Run("no tests returns empty", func(t *testing.T) { img := projectconfig.ImageConfig{} - assert.Empty(t, img.TestNames()) + assert.Empty(t, img.TestRefNames()) }) } -func TestTestSuiteConfig_Validate(t *testing.T) { +func TestTestConfig_Validate(t *testing.T) { t.Run("valid pytest config", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -87,7 +87,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { projectconfig.PytestInstallNone, } { t.Run(string(mode), func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -101,7 +101,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("valid pytest config with empty install mode", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -112,7 +112,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("invalid install mode", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -131,7 +131,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { projectconfig.PytestInstallRequirements, } { t.Run(string(mode), func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -148,7 +148,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("install none without working-dir is valid", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -160,7 +160,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { t.Run("default install without working-dir is valid", func(t *testing.T) { // Default mode is 'none' (no install) and so doesn't require working-dir. - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ @@ -171,7 +171,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("pytest missing subtable", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, } @@ -182,7 +182,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("valid lisa type", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "vm-tests", Type: projectconfig.TestTypeLisa, } @@ -190,7 +190,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("valid lisa type with description", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "vm-tests", Description: "VM integration tests using LISA", Type: projectconfig.TestTypeLisa, @@ -199,7 +199,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("lisa rejects pytest subtable", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "vm-tests", Type: projectconfig.TestTypeLisa, Pytest: &projectconfig.PytestConfig{ @@ -215,7 +215,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("unknown test type", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "bad", Type: "unknown-type", } @@ -225,7 +225,7 @@ func TestTestSuiteConfig_Validate(t *testing.T) { }) t.Run("missing type returns missing-field error", func(t *testing.T) { - testConfig := projectconfig.TestSuiteConfig{ + testConfig := projectconfig.TestConfig{ Name: "smoke", // Type intentionally omitted. } @@ -234,6 +234,45 @@ func TestTestSuiteConfig_Validate(t *testing.T) { require.ErrorIs(t, err, projectconfig.ErrMissingTestField) assert.Contains(t, err.Error(), "type") }) + + t.Run("valid kinds", func(t *testing.T) { + testConfig := projectconfig.TestConfig{ + Name: "smoke", + Type: projectconfig.TestTypeLisa, + Kind: []projectconfig.TestKind{ + projectconfig.TestKindFunctional, + projectconfig.TestKindPerformance, + }, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("unknown kind", func(t *testing.T) { + testConfig := projectconfig.TestConfig{ + Name: "smoke", + Type: projectconfig.TestTypeLisa, + Kind: []projectconfig.TestKind{"bogus"}, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUnknownTestKind) + assert.Contains(t, err.Error(), "bogus") + }) + + t.Run("duplicate kind", func(t *testing.T) { + testConfig := projectconfig.TestConfig{ + Name: "smoke", + Type: projectconfig.TestTypeLisa, + Kind: []projectconfig.TestKind{ + projectconfig.TestKindFunctional, + projectconfig.TestKindFunctional, + }, + } + err := testConfig.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate kind") + assert.Contains(t, err.Error(), string(projectconfig.TestKindFunctional)) + }) } func TestPytestConfig_EffectiveInstallMode(t *testing.T) { @@ -253,16 +292,16 @@ func TestPytestConfig_EffectiveInstallMode(t *testing.T) { }) } -func TestTestSuiteConfig_MergeUpdatesFrom(t *testing.T) { +func TestTestConfig_MergeUpdatesFrom(t *testing.T) { t.Run("merge overrides non-zero fields", func(t *testing.T) { - base := projectconfig.TestSuiteConfig{ + base := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ WorkingDir: "tests", }, } - other := projectconfig.TestSuiteConfig{ + other := projectconfig.TestConfig{ Description: "Updated description", } require.NoError(t, base.MergeUpdatesFrom(&other)) @@ -271,14 +310,14 @@ func TestTestSuiteConfig_MergeUpdatesFrom(t *testing.T) { }) t.Run("merge appends test-paths", func(t *testing.T) { - base := projectconfig.TestSuiteConfig{ + base := projectconfig.TestConfig{ Name: "smoke", Type: projectconfig.TestTypePytest, Pytest: &projectconfig.PytestConfig{ TestPaths: []string{"cases/"}, }, } - other := projectconfig.TestSuiteConfig{ + other := projectconfig.TestConfig{ Pytest: &projectconfig.PytestConfig{ TestPaths: []string{"extra/"}, }, @@ -288,16 +327,16 @@ func TestTestSuiteConfig_MergeUpdatesFrom(t *testing.T) { }) } -func TestValidateTestSuiteReferences(t *testing.T) { +func TestValidateTestReferences(t *testing.T) { t.Run("valid references", func(t *testing.T) { cfg := projectconfig.ProjectConfig{ Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, + Tests: projectconfig.ImageTestsConfig{Tests: []projectconfig.TestRef{{Name: "smoke"}}}, }, }, - TestSuites: map[string]projectconfig.TestSuiteConfig{ + Tests: map[string]projectconfig.TestConfig{ "smoke": { Name: "smoke", Type: projectconfig.TestTypePytest, @@ -320,10 +359,10 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, + Tests: projectconfig.ImageTestsConfig{Tests: []projectconfig.TestRef{{Name: "nonexistent"}}}, }, }, - TestSuites: make(map[string]projectconfig.TestSuiteConfig), + Tests: make(map[string]projectconfig.TestConfig), Components: make(map[string]projectconfig.ComponentConfig), ComponentGroups: make(map[string]projectconfig.ComponentGroupConfig), Distros: make(map[string]projectconfig.DistroDefinition), @@ -332,7 +371,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { } err := cfg.Validate() require.Error(t, err) - require.ErrorIs(t, err, projectconfig.ErrUndefinedTestSuite) + require.ErrorIs(t, err, projectconfig.ErrUndefinedTest) assert.Contains(t, err.Error(), "nonexistent") }) @@ -341,7 +380,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": {Name: "myimage"}, }, - TestSuites: make(map[string]projectconfig.TestSuiteConfig), + Tests: make(map[string]projectconfig.TestConfig), Components: make(map[string]projectconfig.ComponentConfig), ComponentGroups: make(map[string]projectconfig.ComponentGroupConfig), Distros: make(map[string]projectconfig.DistroDefinition), diff --git a/internal/projectconfig/testgroup.go b/internal/projectconfig/testgroup.go new file mode 100644 index 00000000..3f9b95b0 --- /dev/null +++ b/internal/projectconfig/testgroup.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + + "dario.cat/mergo" +) + +var ( + // ErrDuplicateTestGroups is returned when duplicate conflicting test-group definitions are found. + ErrDuplicateTestGroups = errors.New("duplicate test-group") + // ErrUndefinedTestGroup is returned when something references a test-group name that is not defined. + ErrUndefinedTestGroup = errors.New("undefined test-group reference") +) + +// TestGroupConfig is a named bundle of tests. Test-groups are referenced from +// images and components via [TestRef] entries with [TestRef.Group] set. A group +// is a stable, named handle for an underlying set of tests, allowing the +// member list to evolve without churning every image/component reference. +// +// Group membership is a single layer of indirection: groups list test names +// only, not other groups. +type TestGroupConfig struct { + // The test-group's name; not present in serialized TOML files (populated from the map key). + Name string `toml:"-" json:"name" table:",sortkey"` + + // Description of the test-group. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test-group"` + + // Tests lists the names of [TestConfig]s that belong to this group. + Tests []string `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Names of tests (keys in [tests]) that belong to this group"` + + // Reference to the source config file that this definition came from; not present + // in serialized files. + SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` +} + +// Validate checks that the test-group has a non-empty name and that the +// members slice does not contain duplicates. Whether the named tests actually +// exist is checked at project level by [validateTestGroupMembership]. +func (g *TestGroupConfig) Validate() error { + seen := make(map[string]bool, len(g.Tests)) + + for _, name := range g.Tests { + if name == "" { + return fmt.Errorf("test-group %#q contains an empty test name", g.Name) + } + + if seen[name] { + return fmt.Errorf("test-group %#q contains duplicate test %#q", g.Name, name) + } + + seen[name] = true + } + + return nil +} + +// MergeUpdatesFrom updates the test-group config with overrides present in other. +func (g *TestGroupConfig) MergeUpdatesFrom(other *TestGroupConfig) error { + err := mergo.Merge(g, other, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + return fmt.Errorf("failed to merge test-group config:\n%w", err) + } + + return nil +} + +// WithAbsolutePaths returns a copy of the test-group config. It exists for +// symmetry with the other config types; [TestGroupConfig] has no path fields, +// so the copy is byte-identical aside from defensive slice cloning. +func (g *TestGroupConfig) WithAbsolutePaths(_ string) *TestGroupConfig { + result := &TestGroupConfig{ + Name: g.Name, + Description: g.Description, + Tests: append([]string(nil), g.Tests...), + SourceConfigFile: g.SourceConfigFile, + } + + return result +} diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go deleted file mode 100644 index d8424592..00000000 --- a/internal/projectconfig/testsuite.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package projectconfig - -import ( - "errors" - "fmt" - - "dario.cat/mergo" -) - -// TestType indicates the type of test framework used to run a test suite. -type TestType string - -const ( - // TestTypePytest uses pytest to run static/offline validation checks. - TestTypePytest TestType = "pytest" - // TestTypeLisa uses LISA (Linux Integration Services Automation) to run VM-level tests. - TestTypeLisa TestType = "lisa" -) - -var ( - // ErrDuplicateTestSuites is returned when duplicate conflicting test suite definitions are found. - ErrDuplicateTestSuites = errors.New("duplicate test suite") - // ErrUnknownTestType is returned for unrecognized test types. - ErrUnknownTestType = errors.New("unknown test type") - // ErrMissingTestField is returned when a required test config field is missing. - ErrMissingTestField = errors.New("missing required test field") - // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. - ErrUndefinedTestSuite = errors.New("undefined test suite reference") - // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not - // match its declared type. - ErrMismatchedTestSubtable = errors.New("mismatched test subtable") - // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. - ErrInvalidInstallMode = errors.New("invalid install mode") -) - -// TestSuiteConfig defines a named test suite. -type TestSuiteConfig struct { - // The test suite's name; not present in serialized TOML files (populated from the map key). - Name string `toml:"-" json:"name" table:",sortkey"` - - // Description of the test suite. - Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test suite"` - - // Type indicates the test framework to use. - Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,enum=lisa,title=Type,description=Type of test framework (pytest or lisa)"` - - // Pytest holds pytest-specific configuration. Required when Type is "pytest". - Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` - - // Reference to the source config file that this definition came from; not present - // in serialized files. - SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` -} - -// PytestInstallMode specifies how Python dependencies are installed for a pytest suite. -type PytestInstallMode string - -const ( - // PytestInstallPyproject installs dependencies from pyproject.toml using editable mode. - // Returns an error if pyproject.toml is not found in the working directory. - PytestInstallPyproject PytestInstallMode = "pyproject" - // PytestInstallRequirements installs dependencies from requirements.txt. - // Returns an error if requirements.txt is not found. - PytestInstallRequirements PytestInstallMode = "requirements" - // PytestInstallNone skips dependency installation entirely. This is the default - // when [PytestConfig.Install] is not specified — pytest must already be available - // in the venv (e.g., pre-installed, or installed by the test author out-of-band). - PytestInstallNone PytestInstallMode = "none" -) - -// PytestConfig holds configuration specific to pytest-based test suites. -type PytestConfig struct { - // WorkingDir is the directory to use as the current working directory when running pytest. - // Relative paths are resolved against the config file's directory. - WorkingDir string `toml:"working-dir,omitempty" json:"workingDir,omitempty" jsonschema:"title=Working directory,description=Directory to use as CWD when running pytest"` - - // TestPaths is the list of test file paths or directories to pass to pytest as positional - // arguments. Glob patterns (e.g., cases/test_*.py) are expanded relative to WorkingDir. - TestPaths []string `toml:"test-paths,omitempty" json:"testPaths,omitempty" jsonschema:"title=Test paths,description=Test file paths or directories passed to pytest. Glob patterns are expanded."` - - // ExtraArgs is the list of additional arguments to pass to pytest. These are passed - // verbatim after placeholder substitution. Use {image-path} as a placeholder for the - // image path, which will be substituted at runtime. - ExtraArgs []string `toml:"extra-args,omitempty" json:"extraArgs,omitempty" jsonschema:"title=Extra arguments,description=Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path."` - - // Install specifies how Python dependencies are installed into the venv before running - // pytest. Defaults to "none" (no install) when not specified. - Install PytestInstallMode `toml:"install,omitempty" json:"install,omitempty" jsonschema:"enum=pyproject,enum=requirements,enum=none,title=Install mode,description=How to install Python dependencies: pyproject\\, requirements\\, or none (default)"` -} - -// Validate checks that the test suite config has valid type-specific required fields and that -// only the matching subtable is present. -func (t *TestSuiteConfig) Validate() error { - if t.Type == "" { - return fmt.Errorf("%w: test suite %#q is missing required field 'type'", - ErrMissingTestField, t.Name) - } - - switch t.Type { - case TestTypePytest: - if t.Pytest == nil { - return fmt.Errorf("%w: test suite %#q of type %#q requires a [pytest] subtable", - ErrMissingTestField, t.Name, t.Type) - } - - if err := t.Pytest.Validate(); err != nil { - return fmt.Errorf("test suite %#q: %w", t.Name, err) - } - - case TestTypeLisa: - // LISA is an external test framework not executed by azldev. - // Suites of this type serve as metadata for external orchestration (e.g. control tower). - if t.Pytest != nil { - return fmt.Errorf( - "%w: test suite %#q of type %#q cannot include subtable 'pytest'", - ErrMismatchedTestSubtable, - t.Name, - t.Type, - ) - } - - default: - return fmt.Errorf("%w: %#q (test suite: %#q)", ErrUnknownTestType, t.Type, t.Name) - } - - return nil -} - -// Validate checks that the [PytestConfig] fields are valid. -func (p *PytestConfig) Validate() error { - if p.Install != "" && !p.Install.isValid() { - return fmt.Errorf( - "%w: %#q; allowed values: %#q, %#q, %#q (or omit for default %#q)", - ErrInvalidInstallMode, p.Install, - PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone, - PytestInstallNone, - ) - } - - // When the effective install mode requires a working directory, 'working-dir' - // must be specified. The default mode is 'none' (no install) and so requires - // nothing; only an explicitly-set install mode that performs work needs the dir. - if p.EffectiveInstallMode() != PytestInstallNone && p.WorkingDir == "" { - return fmt.Errorf( - "%w: 'working-dir' is required when install mode is %#q", - ErrMissingTestField, p.EffectiveInstallMode(), - ) - } - - return nil -} - -// EffectiveInstallMode returns the install mode, defaulting to [PytestInstallNone] when -// the field is not set. -func (p *PytestConfig) EffectiveInstallMode() PytestInstallMode { - if p.Install == "" { - return PytestInstallNone - } - - return p.Install -} - -// isValid returns whether the mode is a recognized [PytestInstallMode] value. -func (m PytestInstallMode) isValid() bool { - switch m { - case PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone: - return true - default: - return false - } -} - -// MergeUpdatesFrom updates the test suite config with overrides present in other. -func (t *TestSuiteConfig) MergeUpdatesFrom(other *TestSuiteConfig) error { - err := mergo.Merge(t, other, mergo.WithOverride, mergo.WithAppendSlice) - if err != nil { - return fmt.Errorf("failed to merge test suite config:\n%w", err) - } - - return nil -} - -// WithAbsolutePaths returns a copy of the test suite config with relative file paths converted -// to absolute paths (relative to referenceDir). -func (t *TestSuiteConfig) WithAbsolutePaths(referenceDir string) *TestSuiteConfig { - result := &TestSuiteConfig{ - Name: t.Name, - Description: t.Description, - Type: t.Type, - SourceConfigFile: t.SourceConfigFile, - } - - if t.Pytest != nil { - result.Pytest = &PytestConfig{ - WorkingDir: makeAbsolute(referenceDir, t.Pytest.WorkingDir), - TestPaths: append([]string(nil), t.Pytest.TestPaths...), - ExtraArgs: append([]string(nil), t.Pytest.ExtraArgs...), - Install: t.Pytest.Install, - } - } - - return result -} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7b824f4e..96de5f31 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Test configuration for this component" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "test-suites": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this component" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -427,11 +446,19 @@ }, "test-suites": { "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" + "$ref": "#/$defs/TestConfig" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of tests for this project" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroupConfig" }, "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" + "title": "Test Groups", + "description": "Definitions of test-groups (named bundles of tests) for this project" } }, "additionalProperties": false, @@ -705,11 +732,11 @@ "properties": { "test-suites": { "items": { - "$ref": "#/$defs/TestSuiteRef" + "$ref": "#/$defs/TestRef" }, "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,12 +1315,44 @@ "subpath" ] }, - "TestSuiteConfig": { + "TestConfig": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "required": [ + "pytest" + ] + } + } + } + ], "properties": { "description": { "type": "string", "title": "Description", - "description": "Description of this test suite" + "description": "Description of this test" }, "type": { "type": "string", @@ -1304,6 +1363,24 @@ "title": "Type", "description": "Type of test framework (pytest or lisa)" }, + "kind": { + "items": { + "type": "string", + "enum": [ + "functional", + "performance" + ] + }, + "type": "array", + "uniqueItems": true, + "title": "Kind", + "description": "Classification of the test's nature (e.g. functional, performance). Closed enum; multi-valued." + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Indicates that this test may take hours to complete. Hint about cost, not a configurable timeout." + }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", @@ -1316,19 +1393,66 @@ "type" ] }, - "TestSuiteRef": { + "TestGroupConfig": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test-group" + }, + "tests": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tests", + "description": "Names of tests (keys in [tests]) that belong to this group" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], "properties": { "name": { "type": "string", + "minLength": 1, + "pattern": "^\\S", "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" + "description": "Name of a test (key in [tests]); mutually exclusive with 'group'" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test-group (key in [test-groups]); mutually exclusive with 'name'" } }, "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] + "type": "object" }, "ToolsConfig": { "properties": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7b824f4e..96de5f31 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Test configuration for this component" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "test-suites": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this component" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -427,11 +446,19 @@ }, "test-suites": { "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" + "$ref": "#/$defs/TestConfig" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of tests for this project" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroupConfig" }, "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" + "title": "Test Groups", + "description": "Definitions of test-groups (named bundles of tests) for this project" } }, "additionalProperties": false, @@ -705,11 +732,11 @@ "properties": { "test-suites": { "items": { - "$ref": "#/$defs/TestSuiteRef" + "$ref": "#/$defs/TestRef" }, "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,12 +1315,44 @@ "subpath" ] }, - "TestSuiteConfig": { + "TestConfig": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "required": [ + "pytest" + ] + } + } + } + ], "properties": { "description": { "type": "string", "title": "Description", - "description": "Description of this test suite" + "description": "Description of this test" }, "type": { "type": "string", @@ -1304,6 +1363,24 @@ "title": "Type", "description": "Type of test framework (pytest or lisa)" }, + "kind": { + "items": { + "type": "string", + "enum": [ + "functional", + "performance" + ] + }, + "type": "array", + "uniqueItems": true, + "title": "Kind", + "description": "Classification of the test's nature (e.g. functional, performance). Closed enum; multi-valued." + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Indicates that this test may take hours to complete. Hint about cost, not a configurable timeout." + }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", @@ -1316,19 +1393,66 @@ "type" ] }, - "TestSuiteRef": { + "TestGroupConfig": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test-group" + }, + "tests": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tests", + "description": "Names of tests (keys in [tests]) that belong to this group" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], "properties": { "name": { "type": "string", + "minLength": 1, + "pattern": "^\\S", "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" + "description": "Name of a test (key in [tests]); mutually exclusive with 'group'" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test-group (key in [test-groups]); mutually exclusive with 'name'" } }, "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] + "type": "object" }, "ToolsConfig": { "properties": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7b824f4e..96de5f31 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Test configuration for this component" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "test-suites": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this component" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -427,11 +446,19 @@ }, "test-suites": { "additionalProperties": { - "$ref": "#/$defs/TestSuiteConfig" + "$ref": "#/$defs/TestConfig" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of tests for this project" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroupConfig" }, "type": "object", - "title": "Test Suites", - "description": "Definitions of test suites for this project" + "title": "Test Groups", + "description": "Definitions of test-groups (named bundles of tests) for this project" } }, "additionalProperties": false, @@ -705,11 +732,11 @@ "properties": { "test-suites": { "items": { - "$ref": "#/$defs/TestSuiteRef" + "$ref": "#/$defs/TestRef" }, "type": "array", - "title": "Test Suites", - "description": "List of test suite references that apply to this image" + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,12 +1315,44 @@ "subpath" ] }, - "TestSuiteConfig": { + "TestConfig": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "required": [ + "pytest" + ] + } + } + } + ], "properties": { "description": { "type": "string", "title": "Description", - "description": "Description of this test suite" + "description": "Description of this test" }, "type": { "type": "string", @@ -1304,6 +1363,24 @@ "title": "Type", "description": "Type of test framework (pytest or lisa)" }, + "kind": { + "items": { + "type": "string", + "enum": [ + "functional", + "performance" + ] + }, + "type": "array", + "uniqueItems": true, + "title": "Kind", + "description": "Classification of the test's nature (e.g. functional, performance). Closed enum; multi-valued." + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Indicates that this test may take hours to complete. Hint about cost, not a configurable timeout." + }, "pytest": { "$ref": "#/$defs/PytestConfig", "title": "Pytest config", @@ -1316,19 +1393,66 @@ "type" ] }, - "TestSuiteRef": { + "TestGroupConfig": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test-group" + }, + "tests": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tests", + "description": "Names of tests (keys in [tests]) that belong to this group" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], "properties": { "name": { "type": "string", + "minLength": 1, + "pattern": "^\\S", "title": "Name", - "description": "Name of the test suite (must match a key in [test-suites])" + "description": "Name of a test (key in [tests]); mutually exclusive with 'group'" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test-group (key in [test-groups]); mutually exclusive with 'name'" } }, "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] + "type": "object" }, "ToolsConfig": { "properties": {