specs: Add spec 083 and the samples store

conformance/spec036_mcp_execute_tool/execute_test.go

@@ -225,6 +225,20 @@ func readRun(t *testing.T, session *mcpsdk.ClientSession, name, dagRunID, subRun
 	return data
 }
 
+// pollRun reads a root run's detail model for a poll, reporting a failed
+// read to c so the poll tries again.
+func pollRun(t *testing.T, c *assert.CollectT, session *mcpsdk.ClientSession, name, dagRunID string) (map[string]any, bool) {
+	result, err := session.CallTool(mcptest.Context(t), &mcpsdk.CallToolParams{
+		Name:      "dagu_read",
+		Arguments: map[string]any{"target": "run", "name": name, "dagRunId": dagRunID},
+	})
+	if !assert.NoError(c, err) || !assert.False(c, result.IsError, "dagu_read failed") {
+		return nil, false
+	}
+	run, ok := mcptest.StructuredMap(t, result)["data"].(map[string]any)
+	return run, assert.True(c, ok)
+}
+
 func readStep(t *testing.T, run map[string]any, name string) map[string]any {
 	t.Helper()
 
@@ -337,7 +351,12 @@ func TestExecuteRefusesDecisionFromEarlierAttempt(t *testing.T) {
 
 	var current string
 	require.EventuallyWithT(t, func(c *assert.CollectT) {
-		run := readRun(t, session, dagName, dagRunID, "")
+		// A read can fail while the run's new attempt starts, so a failed
+		// read is polled again rather than failing the test.
+		run, ok := pollRun(t, c, session, dagName, dagRunID)
+		if !ok {
+			return
+		}
 		current, _ = run["attemptId"].(string)
 		assert.NotEqual(c, earlier, current)
 		assert.Equal(c, "waiting", readStep(t, run, "deploy_gate")["statusLabel"])

conformance/spec054_template/template_test.go

@@ -28,6 +28,7 @@ func TestTemplateErrors(t *testing.T) {
 		{"start", "missingkey.yaml", "map has no entry for key"},
 		{"validate", "parse_error.yaml", "template: parse error"},
 		{"validate", "missing_dot.yaml", `"report" is a with.data key and needs a leading dot`},
+		{"validate", "unknown_field.yaml", `unexpected additional properties ["datas"]; did you mean "data" instead of "datas"?`},
 	} {
 		t.Run(tc.file, func(t *testing.T) {
 			t.Parallel()

conformance/spec054_template/testdata/unknown_field.yaml

@@ -0,0 +1,5 @@
+steps:
+  - action: template.render
+    with:
+      template: "hi"
+      datas: {}

conformance/spec072_browser/browser_test.go

@@ -9,6 +9,7 @@ package spec072_browser_test
 import (
 	"encoding/json"
 	"fmt"
+	"html"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -18,6 +19,7 @@ import (
 	"regexp"
 	"runtime"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -96,6 +98,49 @@ document.getElementById('screen').textContent = screen.width >= outerWidth && sc
 
 const layoutBanner = `<button onclick="document.getElementById('status').textContent='Banner closed'">Dismiss</button>`
 
+// bidsPage is a site built of frames, as older ones are. Its Search button
+// writes a "searching" window in place and posts into the results frame,
+// whose answer comes after bidsAnswerDelay and closes the window as it
+// loads. A search with no region is answered at once with an error that
+// leaves the window open, as such sites do. Choose region opens a window
+// to tick regions in, which stays until Register gives them to the search
+// and closes it.
+const bidsPage = `<!doctype html><html><head><title>Bids</title><script>
+function openSearching() {
+  top.searching = window.open("/bids/blank", "searching", "width=320,height=120");
+  top.searching.document.write("<title>Searching</title><p>Searching, please wait</p><input name='indicator'>");
+  top.searching.document.close();
+}
+function closeSearching() { if (top.searching) top.searching.close(); }
+</script></head>
+<frameset cols="200,*"><frame src="/bids/menu"><frame name="results" src="/bids/home"></frameset></html>`
+
+const bidsMenuPage = `<!doctype html><html><body>
+<form id="search" method="post" action="/bids/results" target="results"><input type="hidden" name="region" id="region"></form>
+<p id="chosen">All regions</p>
+<button onclick="top.openSearching(); document.getElementById('search').submit()">Search</button>
+<button onclick="window.open('/bids/regions', 'regions', 'width=800,height=600')">Choose region</button>
+<script>function setRegion(name) { document.getElementById('region').value = name; document.getElementById('chosen').textContent = name; }</script>
+</body></html>`
+
+const bidsRegionsPage = `<!doctype html><html><head><title>Regions</title></head><body>
+<label><input type="checkbox" value="Chiba">Chiba</label>
+<label><input type="checkbox" value="Funabashi">Funabashi</label>
+<button onclick="opener.setRegion([...document.querySelectorAll('input:checked')].map(box => box.value).join(', ')); window.close()">Register</button>
+</body></html>`
+
+const bidsErrorPage = `<!doctype html><html><body><p>Input error: choose a region</p></body></html>`
+
+const bidsResultsPage = `<!doctype html><html><body onload="top.closeSearching()">
+<h1>Road repair</h1>
+<p>Road repair in %s</p>
+<button onclick="this.textContent = 'Opened'">Open</button>
+</body></html>`
+
+// bidsAnswerDelay outlasts the five seconds a page gets to settle after an
+// action.
+const bidsAnswerDelay = 7 * time.Second
+
 // browserCommandTimeout bounds a command that starts a browser. Starting
 // one takes tens of seconds when the conformance job loads the runner.
 const browserCommandTimeout = 2 * time.Minute
@@ -116,7 +161,7 @@ const (
 )
 
 var (
-	buttonPattern    = regexp.MustCompile(`\[(\d+-\d+)\] (?:button|link): ([^\n]+)`)
+	buttonPattern    = regexp.MustCompile(`\[(\d+-\d+)\] (?:button|link|checkbox): ([^\n]+)`)
 	quotedPattern    = regexp.MustCompile(`'([^']+)'`)
 	orderPattern     = regexp.MustCompile(`Order confirmed: (\S+)`)
 	headingPattern   = regexp.MustCompile(`heading: ([^\n]+)`)
@@ -239,6 +284,7 @@ func (m *scriptedModel) serve(w http.ResponseWriter, r *http.Request) {
 
 func startShop(t *testing.T) string {
 	t.Helper()
+	items := &itemPages{}
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/rows" {
 			w.Header().Set("Content-Type", "text/html")
@@ -282,6 +328,14 @@ func startShop(t *testing.T) string {
 <script>setTimeout(() => { document.cookie = "passed=1; path=/"; location.reload(); }, 1000);</script>`)
 			return
 		}
+		if strings.HasPrefix(r.URL.Path, "/bids") {
+			serveBids(w, r)
+			return
+		}
+		if strings.HasPrefix(r.URL.Path, "/items/") {
+			items.serve(w, r)
+			return
+		}
 		if r.URL.Path == "/agent" {
 			sent, _ := json.Marshal(r.UserAgent())
 			w.Header().Set("Content-Type", "text/html")
@@ -301,6 +355,57 @@ func startShop(t *testing.T) string {
 	return server.URL
 }
 
+// itemPages serves a page per foreach item at /items/page?item=N, and at
+// /items/turn?item=N answers 200 once the page of item N-1 has been
+// served, so an item's turn comes while the item before it shows its page.
+type itemPages struct {
+	mu     sync.Mutex
+	served map[string]bool
+}
+
+func (p *itemPages) serve(w http.ResponseWriter, r *http.Request) {
+	item := r.URL.Query().Get("item")
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	switch r.URL.Path {
+	case "/items/page":
+		if p.served == nil {
+			p.served = map[string]bool{}
+		}
+		p.served[item] = true
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = fmt.Fprintf(w, "<!doctype html><title>Item %s</title><p>Item %s</p>", html.EscapeString(item), html.EscapeString(item))
+	case "/items/turn":
+		index, err := strconv.Atoi(item)
+		if err != nil || index > 0 && !p.served[strconv.Itoa(index-1)] {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+	}
+}
+
+// serveBids serves bidsPage and its frames and windows.
+func serveBids(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/html")
+	switch r.URL.Path {
+	case "/bids":
+		_, _ = io.WriteString(w, bidsPage)
+	case "/bids/menu":
+		_, _ = io.WriteString(w, bidsMenuPage)
+	case "/bids/regions":
+		_, _ = io.WriteString(w, bidsRegionsPage)
+	case "/bids/home":
+		_, _ = io.WriteString(w, "<p>Choose a region and search</p>")
+	case "/bids/results":
+		region := r.PostFormValue("region")
+		if region == "" {
+			_, _ = io.WriteString(w, bidsErrorPage)
+			return
+		}
+		time.Sleep(bidsAnswerDelay)
+		_, _ = fmt.Fprintf(w, bidsResultsPage, html.EscapeString(region))
+	}
+}
+
 // requireChrome skips when no Chrome is installed, except in CI, where the
 // runners provide one.
 func requireChrome(t *testing.T) {
@@ -438,6 +543,17 @@ func TestBrowserDownload(t *testing.T) {
 	b.dagu.ExpectTextFileContent("report.out", reportBody)
 }
 
+// A goto to a file the site answers as a download ends as soon as the
+// download begins, rather than waiting for a page that never comes, and
+// the file is saved in the run artifacts.
+func TestBrowserGotoDownload(t *testing.T) {
+	t.Parallel()
+
+	b := newBrowserEnv(t)
+	b.dagu.RunWithEnv(b.env, "start", "download_goto.yaml").ExpectExitCode(0)
+	b.dagu.ExpectTextFileContent("report.out", reportBody)
+}
+
 // A page that opens dialogs keeps working: the alert shown while it loads
 // and the confirm raised by an act are both accepted.
 func TestBrowserDialogs(t *testing.T) {
@@ -496,6 +612,53 @@ func TestBrowserSandboxNotSilentlyOff(t *testing.T) {
 	require.Zero(t, b.model.total())
 }
 
+// A search that opens a "searching" window, which its answer closes, is
+// waited out even when the answer takes longer than a page gets to settle,
+// so the when, expect, extract, and act after it work on the results, not
+// on the window. A window that stays, as one to tick regions in and
+// register, is worked on as a page.
+func TestBrowserSearchingWindow(t *testing.T) {
+	t.Parallel()
+
+	b := newBrowserEnv(t)
+	b.dagu.RunWithEnv(b.env, "start", "searching_window.yaml").ExpectExitCode(0)
+	b.dagu.ExpectTextFileContent("heading.out", "Road repair\n")
+}
+
+// A step that fails on a searching window the site left open over its
+// error names the page behind the window and captures it.
+func TestBrowserNamesThePageBehindAWindowLeftOpen(t *testing.T) {
+	t.Parallel()
+
+	b := newBrowserEnv(t)
+	result := b.dagu.RunWithEnv(b.env, "start", "searching_window_error.yaml")
+	result.ExpectNonZeroExitCode()
+	result.ExpectStderrContains(`the page read is a window that the page "Bids" opened and that is still open in front of it`, "failure-opener.png shows that page")
+}
+
+// The items of a foreach each drive a browser of their own: an item whose
+// browser step starts while another item's browser is open leaves that
+// browser alone, though both run the same step of the same DAG run.
+func TestBrowserForeachItemsKeepTheirBrowsers(t *testing.T) {
+	t.Parallel()
+
+	b := newBrowserEnv(t)
+	b.dagu.RunWithEnv(b.env, "start", "foreach_items.yaml").ExpectExitCode(0)
+	data, err := os.ReadFile(b.dagu.ProjectPath("each.out"))
+	require.NoError(t, err)
+	var aggregate struct {
+		Summary struct {
+			Succeeded int `json:"succeeded"`
+		} `json:"summary"`
+		Items []struct {
+			Status string `json:"status"`
+			Error  string `json:"error"`
+		} `json:"items"`
+	}
+	require.NoError(t, json.Unmarshal(data, &aggregate))
+	require.Equal(t, 2, aggregate.Summary.Succeeded, "items: %+v", aggregate.Items)
+}
+
 func TestBrowserExtract(t *testing.T) {
 	t.Parallel()
 

conformance/spec072_browser/testdata/download_goto.yaml

@@ -0,0 +1,14 @@
+working_dir: .
+env:
+  - SHOP_URL: ${SHOP_URL}
+steps:
+  - id: fetch
+    action: browser.run
+    with:
+      url: ${SHOP_URL}
+      do:
+        - goto: ${SHOP_URL}/report.csv
+          timeout: 30s
+  - id: check
+    depends: fetch
+    run: cat "$DAG_RUN_ARTIFACTS_DIR/browser/fetch/downloads/report.csv" > report.out

conformance/spec072_browser/testdata/foreach_items.yaml

@@ -0,0 +1,31 @@
+working_dir: .
+env:
+  - SHOP_URL: ${SHOP_URL}
+steps:
+  - id: each
+    foreach:
+      # The first item keeps its browser open while the second starts its
+      # own.
+      items: [8s, 1s]
+      max_concurrent: 2
+      steps:
+        - id: turn
+          action: wait.http
+          with:
+            url: ${SHOP_URL}/items/turn?item=${foreach.index}
+            poll_interval: 200ms
+        - id: look
+          depends: turn
+          action: browser.run
+          with:
+            url: ${SHOP_URL}/items/page?item=${foreach.index}
+            do:
+              - wait: {duration: "${foreach.item}"}
+              - expect: {text: "Item ${foreach.index}"}
+    output: EACH
+  - id: record
+    depends: each
+    action: file.write
+    with:
+      path: each.out
+      content: ${EACH}

conformance/spec072_browser/testdata/searching_window.yaml

@@ -0,0 +1,33 @@
+working_dir: .
+env:
+  - SHOP_URL: ${SHOP_URL}
+  - LLM_BASE_URL: ${LLM_BASE_URL}
+llm:
+  provider: local
+  model: test-model
+  base_url: ${LLM_BASE_URL}
+steps:
+  - id: search
+    action: browser.run
+    with:
+      url: ${SHOP_URL}/bids
+      do:
+        - act: Click the Choose region button
+        - act: Tick the Funabashi checkbox
+        - act: Click the Register button
+        - expect: {text: Funabashi}
+        - act: Click the Search button
+        - when: The page shows 'Road repair'
+          expect: The page shows 'Road repair in Funabashi'
+        - extract:
+            instruction: The page heading
+            schema:
+              type: object
+              properties:
+                heading:
+                  type: string
+        - act: Click the Open button
+        - expect: {text: Opened}
+  - id: record
+    depends: search
+    run: echo "${steps.search.outputs.heading}" > heading.out

conformance/spec072_browser/testdata/searching_window_error.yaml

@@ -0,0 +1,16 @@
+working_dir: .
+env:
+  - SHOP_URL: ${SHOP_URL}
+  - LLM_BASE_URL: ${LLM_BASE_URL}
+llm:
+  provider: local
+  model: test-model
+  base_url: ${LLM_BASE_URL}
+steps:
+  - id: search
+    action: browser.run
+    with:
+      url: ${SHOP_URL}/bids
+      do:
+        - act: Click the Search button
+        - expect: {text: Road repair, within: 2s}

conformance/spec078_js_run/js_run_test.go

@@ -83,6 +83,7 @@ func TestJSRunConfig(t *testing.T) {
 		{"missing_script.yaml", "with.script is required"},
 		{"both_inputs.yaml", "does not allow both with.input and with.input_file"},
 		{"syntax_error.yaml", "js: compile error"},
+		{"unknown_field.yaml", `unexpected additional properties ["inputs"]; did you mean "input" instead of "inputs"?`},
 	} {
 		t.Run(tc.fixture, func(t *testing.T) {
 			t.Parallel()

conformance/spec078_js_run/testdata/unknown_field.yaml

@@ -0,0 +1,5 @@
+steps:
+  - action: js.run
+    with:
+      script: return input
+      inputs: {n: "5"}