From 3bce58afb0dfd97607c54171fba72937eadfecc7 Mon Sep 17 00:00:00 2001 From: Sandeep Karambelkar Date: Mon, 11 May 2026 01:40:28 +0530 Subject: [PATCH 1/2] Add lisa test suite type which will be executed by validation service --- docs/user/reference/config/test-suites.md | 13 ++++++++++++- internal/app/azldev/cmds/image/test.go | 4 ++++ internal/projectconfig/testsuite.go | 15 +++++++-------- internal/projectconfig/testsuite_test.go | 17 +++++++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/user/reference/config/test-suites.md b/docs/user/reference/config/test-suites.md index c3826ba1..83f50e39 100644 --- a/docs/user/reference/config/test-suites.md +++ b/docs/user/reference/config/test-suites.md @@ -9,7 +9,7 @@ Test suite names must be simple identifiers (no path separators, traversal segme | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Description | `description` | string | No | Human-readable description of the test suite | -| Type | `type` | string | Yes | Test framework to use. Currently only `"pytest"` is supported. | +| Type | `type` | string | Yes | Test framework to use: `"pytest"` or `"lisa"`. | | Pytest | `pytest` | table | When `type = "pytest"` | Pytest-specific configuration (see below) | Test suites are referenced by images through the [`[images..tests]`](images.md#image-tests) subtable. Each image can reference one or more test suites by name. @@ -47,6 +47,7 @@ The following placeholders may appear in `extra-args` and are substituted at run | `{image-name}` | Name of the image being tested | | `{capabilities}` | Comma-separated list of capability names enabled on the image | + ## Examples ### Basic pytest suite @@ -99,6 +100,16 @@ type = "pytest" test-paths = ["/opt/preinstalled-tests/test_*.py"] ``` +### LISA suite (external) + +LISA test suites are metadata-only — they are not executed by `azldev` but are used by external orchestration systems. + +```toml +[test-suites.vm-integration] +description = "VM integration tests using LISA" +type = "lisa" +``` + ### Referencing test suites from an image ```toml diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 77a6215a..9ea9ad54 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -257,6 +257,10 @@ func runTestSuite( case projectconfig.TestTypePytest: return RunPytestSuite(env, suiteConfig, imageConfig, options) + case projectconfig.TestTypeLisa: + return fmt.Errorf("LISA test suites cannot be run locally via 'azldev image test'; "+ + "test suite %#q must be run through the LISA infrastructure", suiteConfig.Name) + default: return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) } diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index a1cc917c..8ecb669a 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -16,6 +16,8 @@ type TestType string const ( // TestTypePytest uses pytest to run static/offline validation checks. TestTypePytest TestType = "pytest" + // TestTypeLisa uses LISA (Linux Integration Services Automation) to run VM-level tests. + TestTypeLisa TestType = "lisa" ) var ( @@ -28,9 +30,7 @@ var ( // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. ErrUndefinedTestSuite = errors.New("undefined test suite reference") // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not - // match its declared type. Currently only one test type (pytest) exists, so this cannot - // trigger yet. When adding a new test type with its own subtable field, add cross-checks - // in [TestSuiteConfig.Validate] to ensure only the matching subtable is populated. + // match its declared type. ErrMismatchedTestSubtable = errors.New("mismatched test subtable") // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. ErrInvalidInstallMode = errors.New("invalid install mode") @@ -45,7 +45,7 @@ type TestSuiteConfig struct { Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test suite"` // Type indicates the test framework to use. - Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,title=Type,description=Type of test framework (pytest)"` + Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,enum=lisa,title=Type,description=Type of test framework (pytest or lisa)"` // Pytest holds pytest-specific configuration. Required when Type is "pytest". Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` @@ -110,10 +110,9 @@ func (t *TestSuiteConfig) Validate() error { return fmt.Errorf("test suite %#q: %w", t.Name, err) } - // NOTE: When adding a new test type with its own subtable field (e.g., Lisa *LisaConfig), - // add a mismatch check here: - // if t.Lisa != nil { return fmt.Errorf("%w: ...", ErrMismatchedTestSubtable) } - // and add the symmetric check in the new type's case branch. + case TestTypeLisa: + // LISA is an external test framework not executed by azldev. + // Suites of this type serve as metadata for external orchestration (e.g. control tower). default: return fmt.Errorf("%w: %#q (test suite: %#q)", ErrUnknownTestType, t.Type, t.Name) diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index 6d5e96e3..6a6d4f24 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -181,6 +181,23 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.Contains(t, err.Error(), "[pytest]") }) + t.Run("valid lisa type", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "vm-tests", + Type: projectconfig.TestTypeLisa, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("valid lisa type with description", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "vm-tests", + Description: "VM integration tests using LISA", + Type: projectconfig.TestTypeLisa, + } + assert.NoError(t, testConfig.Validate()) + }) + t.Run("unknown test type", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ Name: "bad", From 4a60dbcc17195cf8ddb03fb2ae1451c7671cfa1c Mon Sep 17 00:00:00 2001 From: Sandeep Karambelkar Date: Mon, 11 May 2026 23:35:19 +0530 Subject: [PATCH 2/2] Integrate review comments and fix the pr checks --- internal/projectconfig/testsuite.go | 8 ++++++++ internal/projectconfig/testsuite_test.go | 16 ++++++++++++++++ ...ontainer_config_generate-schema_stdout_1.snap | 5 +++-- ...napshots_config_generate-schema_stdout_1.snap | 5 +++-- schemas/azldev.schema.json | 5 +++-- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index 8ecb669a..d8424592 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -113,6 +113,14 @@ func (t *TestSuiteConfig) Validate() error { case TestTypeLisa: // LISA is an external test framework not executed by azldev. // Suites of this type serve as metadata for external orchestration (e.g. control tower). + if t.Pytest != nil { + return fmt.Errorf( + "%w: test suite %#q of type %#q cannot include subtable 'pytest'", + ErrMismatchedTestSubtable, + t.Name, + t.Type, + ) + } default: return fmt.Errorf("%w: %#q (test suite: %#q)", ErrUnknownTestType, t.Type, t.Name) diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index 6a6d4f24..e24f758f 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -198,6 +198,22 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.NoError(t, testConfig.Validate()) }) + t.Run("lisa rejects pytest subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "vm-tests", + Type: projectconfig.TestTypeLisa, + Pytest: &projectconfig.PytestConfig{ + WorkingDir: "tests", + }, + } + + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "vm-tests") + assert.Contains(t, err.Error(), string(projectconfig.TestTypeLisa)) + }) + t.Run("unknown test type", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ Name: "bad", diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 444f204e..1945915b 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -934,10 +934,11 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest", + "lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig", diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 444f204e..1945915b 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -934,10 +934,11 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest", + "lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig", diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 444f204e..1945915b 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -934,10 +934,11 @@ "type": { "type": "string", "enum": [ - "pytest" + "pytest", + "lisa" ], "title": "Type", - "description": "Type of test framework (pytest)" + "description": "Type of test framework (pytest or lisa)" }, "pytest": { "$ref": "#/$defs/PytestConfig",