feat: add azd CLI evaluation and testing framework

.github/workflows/eval-unit.yml

@@ -0,0 +1,61 @@
+name: "Eval: Unit Tests"
+
+on:
+  pull_request:
+    paths:
+      - "cli/azd/test/eval/**"
+      - "cli/azd/internal/mcp/**"
+      - "cli/azd/cmd/mcp.go"
+      - "cli/azd/cmd/root.go"
+
+permissions:
+  contents: read
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "cli/azd/go.mod"
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Build azd
+        working-directory: cli/azd
+        run: go build -o ./azd .
+
+      - name: Install eval dependencies
+        working-directory: cli/azd/test/eval
+        run: npm ci
+
+      - name: Run unit tests
+        working-directory: cli/azd/test/eval
+        run: npm run test:unit -- --ci
+
+      # Waza YAML validation is local-only (run `npm run waza:validate` locally).
+      # waza CLI is not available in CI.
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install grader test dependencies
+        run: pip install pytest
+
+      - name: Run grader tests
+        working-directory: cli/azd/test/eval/graders
+        run: python -m pytest test_graders.py -v
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-unit-results
+          path: cli/azd/test/eval/reports/
+          retention-days: 30

cli/azd/.vscode/cspell.yaml

@@ -377,6 +377,8 @@ overrides:
   - filename: pkg/project/project.go
     words:
       - copylocks
+      - covdata
+      - GOWORK
   - filename: pkg/ext/models.go
     words:
       - unvalidated
@@ -386,6 +388,28 @@ overrides:
     words:
       - covdata
       - GOWORK
+  - filename: test/eval/README.md
+    words:
+      - Waza
+      - waza
+      - urlopen
+  - filename: "test/eval/graders/*.py"
+    words:
+      - Waza
+      - waza
+      - hdrs
+      - mysite
+      - mydb
+  - filename: "test/eval/tasks/**/*.yaml"
+    words:
+      - authenticat
+      - idempoten
+  - filename: "test/eval/tests/human/*.test.ts"
+    words:
+      - compdef
+      - badcfg
+      - provison
+      - notacommand
 ignorePaths:
   - "**/*_test.go"
   - "**/mock*.go"

cli/azd/pkg/azdext/process_darwin.go

@@ -66,7 +66,7 @@ func findProcessByNameOS(name string) []ProcessInfo {
 	nameLower := strings.ToLower(name)
 	var results []ProcessInfo
 
-	for _, line := range strings.Split(string(output), "\n") {
+	for line := range strings.SplitSeq(string(output), "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
 			continue

cli/azd/test/eval/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+reports/*.json
+reports/*.md
+reports/junit.xml
+!reports/.gitkeep