diff --git a/docs/adr/54378-split-import-field-extractor-by-concern.md b/docs/adr/54378-split-import-field-extractor-by-concern.md new file mode 100644 index 00000000000..89d9be5fe2d --- /dev/null +++ b/docs/adr/54378-split-import-field-extractor-by-concern.md @@ -0,0 +1,50 @@ +# ADR-54378: Split import_field_extractor.go into Concern-Focused Files + +**Date**: 2026-08-21 +**Status**: Accepted +**Deciders**: pelikhan, app/copilot-swe-agent + +--- + +### Context + +`pkg/parser/import_field_extractor.go` had grown to 1,048 lines and 51 functions, all in a single file within the `parser` package. The functions implemented five distinct concerns (engine config, activation/auth fields, scalar/builder config, step/job fields, and model normalization) that were interleaved without structural separation. A DeepReport monolithic-file analysis flagged it as a high-value refactor target. Go's package model allows a single package to span multiple files with full access to unexported identifiers, making a file-level split feasible without any API changes. + +### Decision + +We will split `import_field_extractor.go` into five files along the existing semantic boundaries already implied by function groupings and doc-comment sections, with zero behavior change. All functions are moved verbatim; per-file import lists are trimmed to only what each file uses. The resulting files are: + +- `import_field_extractor.go` — core `importAccumulator` struct, constructor, main pipeline (`extractAllImportFields`, `prepareFrontmatter`), and result building +- `import_field_extractor_engine.go` — engine config and scalar/builder fields (max-turns, mcp-servers, safe-outputs, sandbox mounts, etc.) +- `import_field_extractor_activation.go` — activation/auth fields (bots, skip-roles/bots, github-token, github-app, checkout) +- `import_field_extractor_steps.go` — step/job/env/labels/cache/features/run-install-scripts/observability extraction +- `import_field_extractor_models.go` — `models` field normalization (aliases, allow/block policies, cost overlays) + +### Alternatives Considered + +#### Alternative 1: Reorganize with comment blocks inside the single file + +Add clearly labeled comment banners (`// ---- Engine config ----`) to group related functions in-place. This preserves contiguous `git blame` history and has zero structural change. However, it does not reduce the cognitive overhead of loading the full 1,000-line file to locate one function group, and IDE navigation still requires scrolling rather than opening a named file. + +#### Alternative 2: Extract into sub-packages under pkg/parser/ + +Create `pkg/parser/extractor/engine`, `pkg/parser/extractor/activation`, etc. This provides the strongest isolation: each sub-package can be tested independently and dependencies are explicit. However, the `importAccumulator` struct is intentionally unexported. Elevating it to a sub-package-visible type would require either exporting it (expanding the public API) or moving the struct into the sub-package (restructuring the main file's data model). Neither change is behavior-neutral. The benefits of sub-package isolation are therefore not available at this stage without additional refactoring scope. + +### Consequences + +#### Positive +- Navigating to a specific concern (e.g., model alias normalization) now requires opening one clearly-named file rather than scanning a 1,000-line file. +- Future additions or removals of field extractors are scoped to a single file, reducing the chance of merge conflicts between concurrent changes. +- Each file's import list accurately documents which standard-library packages that concern depends on. + +#### Negative +- The `importAccumulator` struct is defined in `import_field_extractor.go` and is an implicit shared dependency across all five files; this coupling is invisible at the call site and must be understood holistically. +- `git log -- pkg/parser/import_field_extractor.go` no longer surfaces history for functions moved into the new files; `git log --follow` or per-file blame is needed to trace pre-split history. + +#### Neutral +- No callers outside the `parser` package are affected; all moved symbols remain unexported. +- Test coverage does not change: existing tests exercise the package API, not individual file boundaries. + +--- + +*Implemented in PR #54378.* diff --git a/pkg/parser/import_field_extractor.go b/pkg/parser/import_field_extractor.go index 197a0946cd2..980c6a14375 100644 --- a/pkg/parser/import_field_extractor.go +++ b/pkg/parser/import_field_extractor.go @@ -242,13 +242,13 @@ func (acc *importAccumulator) extractToolsContent(rawContent string, item import if wasSubstituted { toolsContent, err := extractToolsFromContent(rawContent) if err != nil { - return "", fmt.Errorf("failed to extract tools from '%s': %w", item.fullPath, err) + return "", fmt.Errorf("could not extract tools from import %q; ensure the imported content has a valid tools block: %w", item.fullPath, err) } return toolsContent, nil } toolsContent, err := processIncludedFileWithVisited(item.fullPath, item.sectionName, true, visited) if err != nil { - return "", fmt.Errorf("failed to process imported file '%s': %w", item.fullPath, err) + return "", fmt.Errorf("could not process import %q for tools extraction; ensure the imported workflow path and section are valid: %w", item.fullPath, err) } return toolsContent, nil } @@ -266,7 +266,7 @@ func (acc *importAccumulator) trackRuntimeOrInlineImport(fullPath, importRelPath parserLog.Printf("Import %s has substituted inputs - will be inlined for compile-time substitution", importRelPath) markdownContent, err := ExtractMarkdownContent(rawContent) if err != nil { - return fmt.Errorf("failed to extract markdown from imported file '%s': %w", fullPath, err) + return fmt.Errorf("could not extract markdown from import %q; ensure the file contains valid markdown after frontmatter: %w", fullPath, err) } appendMarkdownWithSeparator(&acc.markdownBuilder, markdownContent) acc.promptImports = append(acc.promptImports, PromptImportEntry{Markdown: markdownContent}) @@ -299,647 +299,6 @@ func parseFrontmatterForExtraction(rawContent string, wasSubstituted bool, origF return reparsed.Frontmatter } -// extractEngineConfig extracts engine-related settings from the imported frontmatter map -// and accumulates them. Engine configs with only `mcp` sub-keys (no `id` or `runtime`) -// are not counted as engine specifications — they carry MCP gateway settings only. -// -// Side effects: acc.engines, acc.mergedEngineMCPToolTimeout, -// acc.mergedEngineMCPSessionTimeout, acc.mergedEngineModel. -func (acc *importAccumulator) extractEngineConfig(fm map[string]any, fullPath string) { - if modelStr, ok := fm["model"].(string); ok && modelStr != "" && acc.mergedEngineModel == "" { - acc.mergedEngineModel = modelStr - parserLog.Printf("Extracted top-level model preference from import %s: %s", fullPath, modelStr) - } - - engineVal, hasEngine := fm["engine"] - if !hasEngine { - return - } - parserLog.Printf("Found engine config in import: %s", fullPath) - - switch v := engineVal.(type) { - case string: - // String engine (e.g. "copilot") — always counts as an engine spec. - if engineJSON, merr := json.Marshal(v); merr == nil { - acc.engines = append(acc.engines, string(engineJSON)) - } - case map[string]any: - // Object engine — extract engine.mcp.* settings first, then decide - // whether to add to engines based on whether an engine ID is present. - if mcpVal, hasMCP := v["mcp"]; hasMCP { - acc.extractEngineMCPSettings(mcpVal, fullPath) - } - // Only add to engines list if this config specifies an actual engine - // (i.e. it carries an 'id' or 'runtime' field). Configs with only - // 'model' or 'mcp' settings are preferences, not engine selections, - // and must not trigger the "multiple engine fields" validation error. - _, hasID := v["id"] - _, hasRuntime := v["runtime"] - if hasID || hasRuntime { - if engineJSON, merr := json.Marshal(v); merr == nil { - acc.engines = append(acc.engines, string(engineJSON)) - } - } else { - // No engine ID or runtime — this is a model/MCP-only preference. - // Extract the model hint (first-wins) so it can be applied to the - // resolved engine after all imports are processed. - if modelStr, ok := v["model"].(string); ok && modelStr != "" { - if acc.mergedEngineModel == "" { - acc.mergedEngineModel = modelStr - parserLog.Printf("Extracted engine.model preference from import %s: %s", fullPath, modelStr) - } - } - } - default: - // Unexpected type — marshal and add to preserve existing behavior. - if engineJSON, merr := json.Marshal(engineVal); merr == nil { - acc.engines = append(acc.engines, string(engineJSON)) - } - } -} - -// extractEngineMCPSettings extracts engine.mcp.tool-timeout and engine.mcp.session-timeout -// from mcpVal (first-wins across all imports). -func (acc *importAccumulator) extractEngineMCPSettings(mcpVal any, fullPath string) { - mcpMap, ok := mcpVal.(map[string]any) - if !ok { - return - } - if acc.mergedEngineMCPToolTimeout == "" { - if ttStr, ok := mcpMap["tool-timeout"].(string); ok && ttStr != "" { - acc.mergedEngineMCPToolTimeout = ttStr - parserLog.Printf("Extracted engine.mcp.tool-timeout from import %s: %s", fullPath, ttStr) - } - } - if acc.mergedEngineMCPSessionTimeout == "" { - if stStr, ok := mcpMap["session-timeout"].(string); ok && stStr != "" { - acc.mergedEngineMCPSessionTimeout = stStr - parserLog.Printf("Extracted engine.mcp.session-timeout from import %s: %s", fullPath, stStr) - } - } -} - -// extractConfigFields extracts scalar and builder-based configuration fields from the -// frontmatter map and writes them into the appropriate accumulator builders and slices. -// -// Side effects: acc.mergedMaxTurns, acc.mergedMaxToolDenials, acc.mergedMaxRuns, acc.mergedMaxAICredits, -// acc.mergedMaxDailyAICredits, acc.mcpServersBuilder, -// acc.safeOutputs, acc.mcpScripts, acc.stepsBuilder, acc.runtimesBuilder, -// acc.servicesBuilder, acc.networkBuilder, acc.permissionsBuilder, -// acc.secretMaskingBuilder. -func (acc *importAccumulator) extractConfigFields(fm map[string]any, fullPath string) { - acc.extractFirstWinsJSONField(fm, fullPath, "max-turns", &acc.mergedMaxTurns) - acc.extractFirstWinsJSONField(fm, fullPath, "max-tool-denials", &acc.mergedMaxToolDenials) - acc.extractFirstWinsJSONField(fm, fullPath, "max-runs", &acc.mergedMaxRuns) - acc.extractFirstWinsJSONField(fm, fullPath, "max-turn-cache-misses", &acc.mergedMaxTurnCacheMisses) - acc.extractFirstWinsJSONField(fm, fullPath, "max-ai-credits", &acc.mergedMaxAICredits) - acc.extractFirstWinsJSONField(fm, fullPath, "max-daily-ai-credits", &acc.mergedMaxDailyAICredits) - - acc.appendJSONBuilderField(fm, "mcp-servers", "{}", &acc.mcpServersBuilder) - acc.plugins = append(acc.plugins, parseStringSliceField(fm["plugins"], false)...) - acc.appendJSONSliceField(fm, "safe-outputs", "{}", &acc.safeOutputs) - acc.appendJSONSliceField(fm, "mcp-scripts", "{}", &acc.mcpScripts) - acc.appendYAMLBuilderField(fm, "steps", &acc.stepsBuilder) - acc.appendJSONBuilderField(fm, "runtimes", "{}", &acc.runtimesBuilder) - acc.appendYAMLBuilderField(fm, "services", &acc.servicesBuilder) - acc.appendJSONBuilderField(fm, "network", "{}", &acc.networkBuilder) - acc.mergeSandboxAgentMounts(fm) - acc.mergeSandboxAgentRuntimeInstall(fm) - acc.appendJSONBuilderField(fm, "permissions", "{}", &acc.permissionsBuilder) - acc.appendJSONBuilderField(fm, "secret-masking", "{}", &acc.secretMaskingBuilder) -} - -func (acc *importAccumulator) mergeSandboxAgentMounts(fm map[string]any) { - sandboxVal, hasSandbox := fm["sandbox"] - if !hasSandbox { - return - } - - sandboxMap, ok := sandboxVal.(map[string]any) - if !ok { - return - } - - agentVal, hasAgent := sandboxMap["agent"] - if !hasAgent { - return - } - - agentMap, ok := agentVal.(map[string]any) - if !ok { - return - } - - mountsVal, hasMounts := agentMap["mounts"] - if !hasMounts { - return - } - - mounts, ok := mountsVal.([]any) - if !ok { - return - } - - for _, mountVal := range mounts { - mount, ok := mountVal.(string) - if !ok || mount == "" { - continue - } - if !acc.sandboxAgentMountsSet[mount] { - acc.sandboxAgentMountsSet[mount] = true - acc.sandboxAgentMounts = append(acc.sandboxAgentMounts, mount) - } - } -} - -// mergeSandboxAgentRuntimeInstall extracts sandbox.agent.runtime-install from an -// imported workflow's frontmatter. False wins: if any import sets runtime-install -// to false the accumulated value becomes false and stays false. -func (acc *importAccumulator) mergeSandboxAgentRuntimeInstall(fm map[string]any) { - // Already locked to false — no need to inspect further imports. - if acc.sandboxAgentRuntimeInstall != nil && !*acc.sandboxAgentRuntimeInstall { - return - } - - sandboxVal, hasSandbox := fm["sandbox"] - if !hasSandbox { - return - } - sandboxMap, ok := sandboxVal.(map[string]any) - if !ok { - return - } - agentVal, hasAgent := sandboxMap["agent"] - if !hasAgent { - return - } - agentMap, ok := agentVal.(map[string]any) - if !ok { - return - } - riVal, hasRI := agentMap["runtime-install"] - if !hasRI { - return - } - ri, ok := riVal.(bool) - if !ok { - return - } - if !ri { - f := false - acc.sandboxAgentRuntimeInstall = &f - } else if acc.sandboxAgentRuntimeInstall == nil { - t := true - acc.sandboxAgentRuntimeInstall = &t - } -} - -func (acc *importAccumulator) extractFirstWinsJSONField(fm map[string]any, fullPath, field string, target *string) { - if *target != "" { - return - } - fieldJSON, err := extractFieldJSONFromMap(fm, field, "") - if err != nil || fieldJSON == "" || fieldJSON == "null" { - return - } - *target = fieldJSON - parserLog.Printf("Extracted %s from import: %s", field, fullPath) -} - -func (acc *importAccumulator) appendJSONBuilderField(fm map[string]any, field, emptyValue string, builder *strings.Builder) { - content, err := extractFieldJSONFromMap(fm, field, emptyValue) - if err != nil || content == "" || content == emptyValue { - return - } - builder.WriteString(content + "\n") -} - -func (acc *importAccumulator) appendJSONSliceField(fm map[string]any, field, emptyValue string, target *[]string) { - content, err := extractFieldJSONFromMap(fm, field, emptyValue) - if err != nil || content == "" || content == emptyValue { - return - } - *target = append(*target, content) -} - -func (acc *importAccumulator) appendYAMLBuilderField(fm map[string]any, field string, builder *strings.Builder) { - content, err := extractYAMLFieldFromMap(fm, field) - if err != nil || content == "" { - return - } - builder.WriteString(content + "\n") -} - -// extractActivationFields extracts activation and authentication-related fields from -// the frontmatter map: bots, skip-roles, skip-bots, skip-if-match, skip-if-no-match, -// top-level ambient-folders, on.github-token, on.github-app, top-level github-app, and checkout. -// -// Side effects: acc.bots, acc.botsSet, acc.skipRoles, acc.skipRolesSet, acc.skipBots, -// acc.skipBotsSet, acc.skipIfMatch, acc.skipIfNoMatch, acc.activationGitHubToken, -// acc.activationGitHubApp, acc.topLevelGitHubApp, acc.checkouts. -func (acc *importAccumulator) extractActivationFields(fm map[string]any, item importQueueItem) { - acc.mergeBots(fm) - acc.mergeSkipRoles(fm) - acc.mergeSkipBots(fm) - acc.mergeAmbientFolders(fm) - acc.extractActivationSkipMatchFields(fm, item.fullPath) - acc.extractActivationGitHubToken(fm, item.fullPath) - acc.extractActivationGitHubAppFields(fm, item.fullPath) - acc.extractCheckoutField(fm, item.fullPath) -} - -func (acc *importAccumulator) mergeBots(fm map[string]any) { - mergeJSONStringListField(fm, "bots", "[]", acc.botsSet, &acc.bots, func(m map[string]any, field string) (string, error) { - return extractFieldJSONFromMap(m, field, "[]") - }) -} - -func (acc *importAccumulator) mergeSkipRoles(fm map[string]any) { - mergeJSONStringListField(fm, "skip-roles", "[]", acc.skipRolesSet, &acc.skipRoles, extractOnSectionFieldFromMap) -} - -func (acc *importAccumulator) mergeSkipBots(fm map[string]any) { - mergeJSONStringListField(fm, "skip-bots", "[]", acc.skipBotsSet, &acc.skipBots, extractOnSectionFieldFromMap) -} - -func (acc *importAccumulator) mergeAmbientFolders(fm map[string]any) { - mergeJSONStringListField(fm, "ambient-folders", "[]", acc.ambientFoldersSet, &acc.ambientFolders, func(m map[string]any, field string) (string, error) { - return extractFieldJSONFromMap(m, field, "[]") - }) -} - -func mergeJSONStringListField( - fm map[string]any, - field, emptyValue string, - seen map[string]bool, - merged *[]string, - extractor func(map[string]any, string) (string, error), -) { - content, err := extractor(fm, field) - if err != nil || content == "" || content == emptyValue { - return - } - var imported []string - if jsonErr := json.Unmarshal([]byte(content), &imported); jsonErr != nil { - return - } - for _, value := range imported { - if !seen[value] { - seen[value] = true - *merged = append(*merged, value) - } - } -} - -func (acc *importAccumulator) extractActivationSkipMatchFields(fm map[string]any, fullPath string) { - if acc.skipIfMatch == "" { - if skipJSON, skipErr := extractOnSectionAnyFieldFromMap(fm, "skip-if-match"); skipErr == nil && skipJSON != "" && skipJSON != "null" { - acc.skipIfMatch = skipJSON - parserLog.Printf("Extracted on.skip-if-match from import: %s", fullPath) - } - } - if acc.skipIfNoMatch == "" { - if skipJSON, skipErr := extractOnSectionAnyFieldFromMap(fm, "skip-if-no-match"); skipErr == nil && skipJSON != "" && skipJSON != "null" { - acc.skipIfNoMatch = skipJSON - parserLog.Printf("Extracted on.skip-if-no-match from import: %s", fullPath) - } - } -} - -func (acc *importAccumulator) extractActivationGitHubToken(fm map[string]any, fullPath string) { - if acc.activationGitHubToken != "" { - return - } - tokenJSON, tokenErr := extractOnSectionAnyFieldFromMap(fm, "github-token") - if tokenErr != nil || tokenJSON == "" || tokenJSON == "null" { - return - } - var token string - if jsonErr := json.Unmarshal([]byte(tokenJSON), &token); jsonErr == nil && token != "" { - acc.activationGitHubToken = token - parserLog.Printf("Extracted on.github-token from import: %s", fullPath) - } -} - -func (acc *importAccumulator) extractActivationGitHubAppFields(fm map[string]any, fullPath string) { - if acc.activationGitHubApp == "" { - if appJSON, appErr := extractOnSectionAnyFieldFromMap(fm, "github-app"); appErr == nil { - if validated := validateGitHubAppJSON(appJSON); validated != "" { - acc.activationGitHubApp = validated - parserLog.Printf("Extracted on.github-app from import: %s", fullPath) - } - } - } - if acc.topLevelGitHubApp == "" { - if appJSON, appErr := extractFieldJSONFromMap(fm, "github-app", ""); appErr == nil { - if validated := validateGitHubAppJSON(appJSON); validated != "" { - acc.topLevelGitHubApp = validated - parserLog.Printf("Extracted top-level github-app from import: %s", fullPath) - } - } - } -} - -func (acc *importAccumulator) extractCheckoutField(fm map[string]any, fullPath string) { - checkoutJSON, checkoutErr := extractFieldJSONFromMap(fm, "checkout", "") - if checkoutErr != nil || checkoutJSON == "" || checkoutJSON == "null" || checkoutJSON == "false" { - return - } - acc.checkouts = append(acc.checkouts, checkoutJSON) - parserLog.Printf("Extracted checkout from import: %s", fullPath) -} - -// extractStepAndJobFields extracts step and job configuration fields from the frontmatter -// map. Environment variable conflict detection is performed: if the same env var is -// defined in two different imports, an error is returned. -// -// Side effects: acc.preStepsBuilder, acc.preAgentStepsBuilder, acc.postStepsBuilder, -// acc.jobsBuilder, acc.envBuilder, acc.envSources. -func (acc *importAccumulator) extractStepAndJobFields(fm map[string]any, importPath string) error { - // Extract pre-steps (prepend in order). - if preStepsContent, err := extractYAMLFieldFromMap(fm, "pre-steps"); err == nil && preStepsContent != "" { - acc.preStepsBuilder.WriteString(preStepsContent + "\n") - } - - // Extract pre-agent-steps (prepend in order). - if preAgentStepsContent, err := extractYAMLFieldFromMap(fm, "pre-agent-steps"); err == nil && preAgentStepsContent != "" { - acc.preAgentStepsBuilder.WriteString(preAgentStepsContent + "\n") - } - - // Extract post-steps (append in order). - if postStepsContent, err := extractYAMLFieldFromMap(fm, "post-steps"); err == nil && postStepsContent != "" { - acc.postStepsBuilder.WriteString(postStepsContent + "\n") - } - - // Extract jobs (append in order; merged into custom jobs map). - if jobsContent, err := extractFieldJSONFromMap(fm, "jobs", "{}"); err == nil && jobsContent != "" && jobsContent != "{}" { - acc.jobsBuilder.WriteString(jobsContent + "\n") - } - - // Extract env (append in order; main workflow env takes precedence). - // Conflicts between two imports are disallowed — only the main workflow may override imported vars. - envContent, err := extractFieldJSONFromMap(fm, "env", "{}") - if err == nil && envContent != "" && envContent != "{}" { - var envMap map[string]any - if jsonErr := json.Unmarshal([]byte(envContent), &envMap); jsonErr == nil { - for key := range envMap { - if existingSource, exists := acc.envSources[key]; exists { - return fmt.Errorf("env variable %q is defined in multiple imports: %q and %q; remove the duplicate definition from one of the imports, or move it to the main workflow to override imported values", key, existingSource, importPath) - } - acc.envSources[key] = importPath - } - acc.envBuilder.WriteString(envContent + "\n") - } - } - - return nil -} - -// extractFeatureAndObservabilityFields extracts labels, cache, feature flags, model -// aliases, the run-install-scripts flag, observability configuration, and excluded-env -// from the frontmatter map. -// -// Side effects: acc.labels, acc.labelsSet, acc.caches, acc.features, acc.models, -// acc.runInstallScripts, acc.observabilityConfigs, acc.excludedEnv, acc.excludedEnvSet. -func (acc *importAccumulator) extractFeatureAndObservabilityFields(fm map[string]any, fullPath string) { - acc.mergeLabels(fm) - acc.appendCacheField(fm) - acc.appendFeaturesField(fm) - acc.appendModelsField(fm, fullPath) - acc.extractRunInstallScripts(fm, fullPath) - acc.appendObservabilityField(fm, fullPath) - acc.mergeExcludedEnv(fm) -} - -func (acc *importAccumulator) mergeExcludedEnv(fm map[string]any) { - mergeJSONStringListField(fm, "excluded-env", "[]", acc.excludedEnvSet, &acc.excludedEnv, func(m map[string]any, field string) (string, error) { - return extractFieldJSONFromMap(m, field, "[]") - }) -} - -func (acc *importAccumulator) mergeLabels(fm map[string]any) { - mergeJSONStringListField(fm, "labels", "[]", acc.labelsSet, &acc.labels, func(m map[string]any, field string) (string, error) { - return extractFieldJSONFromMap(m, field, "[]") - }) -} - -func (acc *importAccumulator) appendCacheField(fm map[string]any) { - if cacheContent, err := extractFieldJSONFromMap(fm, "cache", "{}"); err == nil && cacheContent != "" && cacheContent != "{}" { - acc.caches = append(acc.caches, cacheContent) - } -} - -func (acc *importAccumulator) appendFeaturesField(fm map[string]any) { - featuresContent, err := extractFieldJSONFromMap(fm, "features", "{}") - if err != nil || featuresContent == "" || featuresContent == "{}" { - return - } - var featuresMap map[string]any - if jsonErr := json.Unmarshal([]byte(featuresContent), &featuresMap); jsonErr == nil { - acc.features = append(acc.features, featuresMap) - parserLog.Printf("Extracted features from import: %d entries", len(featuresMap)) - } -} - -func (acc *importAccumulator) appendModelsField(fm map[string]any, importPath string) { - modelsContent, err := extractFieldJSONFromMap(fm, "models", "{}") - if err != nil || modelsContent == "" || modelsContent == "{}" { - return - } - var rawModels map[string]any - if jsonErr := json.Unmarshal([]byte(modelsContent), &rawModels); jsonErr != nil { - acc.warnings = append(acc.warnings, fmt.Sprintf("import %q: models field is not a valid object; skipping invalid value", importPath)) - return - } - if modelPolicy := normalizeModelPolicies(rawModels, importPath, &acc.warnings); len(modelPolicy) > 0 { - acc.modelPolicies = append(acc.modelPolicies, modelPolicy) - parserLog.Printf("Extracted model policy from import: allowed=%d, blocked=%d", len(modelPolicy["allowed"]), len(modelPolicy["blocked"])) - } - if providers, hasProviders := rawModels["providers"]; hasProviders { - if providerMap, ok := sanitizeModelProvidersForCosts(providers, importPath, &acc.warnings); ok { - acc.modelCosts = append(acc.modelCosts, map[string]any{"providers": providerMap}) - parserLog.Printf("Extracted model costs from import: providers=%d", len(providerMap)) - } - } - if acc.defaultAiCreditsPricing == nil { - if defaultPricing, hasDefaultPricing := rawModels["default-ai-credits-pricing"]; hasDefaultPricing { - if pricingMap, ok := defaultPricing.(map[string]any); ok { - acc.defaultAiCreditsPricing = maps.Clone(pricingMap) - parserLog.Printf("Extracted default-ai-credits-pricing from import: %s", importPath) - } else { - acc.warnings = append(acc.warnings, fmt.Sprintf("import %q: models.default-ai-credits-pricing must be an object; skipping invalid value", importPath)) - } - } - } - - aliasModels := make(map[string]any, len(rawModels)) - for key, value := range rawModels { - // providers is reserved for model-cost overlays and should not be treated - // as an alias key, even when aliases and providers coexist. - if key == "providers" || key == "default-ai-credits-pricing" || isModelPolicyKey(key) { - continue - } - aliasModels[key] = value - } - if len(aliasModels) == 0 { - return - } - modelsMap := normalizeModelAliases(aliasModels) - if len(modelsMap) > 0 { - acc.models = append(acc.models, modelsMap) - parserLog.Printf("Extracted model aliases from import: %d entries", len(modelsMap)) - } -} - -func normalizeModelPolicies(rawModels map[string]any, importPath string, warnings *[]string) map[string][]string { - parse := func(key string) []string { - value, exists := rawModels[key] - if !exists { - return nil - } - return parseModelPolicyField(value, key, importPath, warnings) - } - allowed := parse(modelPolicyAllowedKey) - blocked := parse(modelPolicyBlockedKey) - if len(allowed) == 0 && len(blocked) == 0 { - return nil - } - return map[string][]string{ - modelPolicyAllowedKey: allowed, - modelPolicyBlockedKey: blocked, - } -} - -func normalizeModelAliases(rawModels map[string]any) map[string][]string { - modelsMap := make(map[string][]string, len(rawModels)) - for k, v := range rawModels { - strs := parseStringSliceField(v, true) - if len(strs) == 0 { - continue - } - modelsMap[k] = strs - } - return modelsMap -} - -// parseModelPolicyField parses one imported models policy field as a string list. -// Invalid field shapes or entries are ignored and appended to warnings. -func parseModelPolicyField(value any, fieldName, importPath string, warnings *[]string) []string { - values, ok := value.([]any) - if !ok { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s must be an array; skipping invalid value", importPath, fieldName)) - return nil - } - result := make([]string, 0, len(values)) - for _, v := range values { - s, ok := v.(string) - if !ok { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s contains a non-string entry; skipping invalid entry", importPath, fieldName)) - continue - } - if s == "" { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s contains an empty string entry; skipping invalid entry", importPath, fieldName)) - continue - } - result = append(result, s) - } - if len(result) == 0 { - return nil - } - return result -} - -// sanitizeModelProvidersForCosts validates models.providers from an import. -// It returns the provider map and true when the input is a non-empty object; otherwise false. -func sanitizeModelProvidersForCosts(providers any, importPath string, warnings *[]string) (map[string]any, bool) { - providerMap, ok := providers.(map[string]any) - if !ok || len(providerMap) == 0 { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers must be a non-empty object; skipping invalid value", importPath)) - return nil, false - } - sanitizedProviders := make(map[string]any, len(providerMap)) - for providerName, providerValue := range providerMap { - if isModelPolicyKey(providerName) || providerName == "blocked" { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers.%s is reserved for policy and ignored in cost data", importPath, providerName)) - continue - } - sanitizedProviders[providerName] = providerValue - } - if len(sanitizedProviders) == 0 { - *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers must contain at least one non-policy provider key", importPath)) - return nil, false - } - return sanitizedProviders, true -} - -func parseStringSliceField(value any, keepEmpty bool) []string { - values, ok := value.([]any) - if !ok { - return nil - } - result := make([]string, 0, len(values)) - for _, v := range values { - if s, ok := v.(string); ok { - if s == "" && !keepEmpty { - continue - } - result = append(result, s) - } - } - if len(result) == 0 { - return nil - } - return result -} - -func isModelPolicyKey(key string) bool { - return key == modelPolicyAllowedKey || key == modelPolicyBlockedKey -} - -func (acc *importAccumulator) extractRunInstallScripts(fm map[string]any, fullPath string) { - if acc.runInstallScripts { - return - } - if hasNodeRuntimeRunInstallScripts(fm) { - acc.runInstallScripts = true - parserLog.Printf("Extracted runtimes.node.run-install-scripts: true from import: %s", fullPath) - } -} - -func hasNodeRuntimeRunInstallScripts(fm map[string]any) bool { - runtimesAny, hasRuntimes := fm["runtimes"] - if !hasRuntimes { - return false - } - runtimesMap, ok := runtimesAny.(map[string]any) - if !ok { - return false - } - nodeAny, hasNode := runtimesMap["node"] - if !hasNode { - return false - } - nodeMap, ok := nodeAny.(map[string]any) - if !ok { - return false - } - rsAny, hasRS := nodeMap["run-install-scripts"] - if !hasRS { - return false - } - rsBool, ok := rsAny.(bool) - return ok && rsBool -} - -func (acc *importAccumulator) appendObservabilityField(fm map[string]any, fullPath string) { - obsContent, obsErr := extractFieldJSONFromMap(fm, "observability", "{}") - if obsErr != nil || obsContent == "" || obsContent == "{}" { - return - } - acc.observabilityConfigs = append(acc.observabilityConfigs, obsContent) - parserLog.Printf("Extracted observability from import: %s", fullPath) -} - // toImportsResult converts the accumulated state to a final ImportsResult. // topologicalOrder is the result from topologicalSortImports. func (acc *importAccumulator) toImportsResult(topologicalOrder []string) *ImportsResult { diff --git a/pkg/parser/import_field_extractor_activation.go b/pkg/parser/import_field_extractor_activation.go new file mode 100644 index 00000000000..d02cb8e9009 --- /dev/null +++ b/pkg/parser/import_field_extractor_activation.go @@ -0,0 +1,106 @@ +// Package parser provides functions for parsing and processing workflow markdown files. +// import_field_extractor_activation.go implements extraction of activation and +// authentication-related fields (bots, skip-roles, skip-bots, skip-if-match, +// skip-if-no-match, ambient-folders, github-token, github-app, checkout) from +// imported frontmatter. +package parser + +import ( + "encoding/json" +) + +// extractActivationFields extracts activation and authentication-related fields from +// the frontmatter map: bots, skip-roles, skip-bots, skip-if-match, skip-if-no-match, +// top-level ambient-folders, on.github-token, on.github-app, top-level github-app, and checkout. +// +// Side effects: acc.bots, acc.botsSet, acc.skipRoles, acc.skipRolesSet, acc.skipBots, +// acc.skipBotsSet, acc.skipIfMatch, acc.skipIfNoMatch, acc.activationGitHubToken, +// acc.activationGitHubApp, acc.topLevelGitHubApp, acc.checkouts. +func (acc *importAccumulator) extractActivationFields(fm map[string]any, item importQueueItem) { + acc.mergeBots(fm) + acc.mergeSkipRoles(fm) + acc.mergeSkipBots(fm) + acc.mergeAmbientFolders(fm) + acc.extractActivationSkipMatchFields(fm, item.fullPath) + acc.extractActivationGitHubToken(fm, item.fullPath) + acc.extractActivationGitHubAppFields(fm, item.fullPath) + acc.extractCheckoutField(fm, item.fullPath) +} + +func (acc *importAccumulator) mergeBots(fm map[string]any) { + mergeJSONStringListField(fm, "bots", "[]", acc.botsSet, &acc.bots, func(m map[string]any, field string) (string, error) { + return extractFieldJSONFromMap(m, field, "[]") + }) +} + +func (acc *importAccumulator) mergeSkipRoles(fm map[string]any) { + mergeJSONStringListField(fm, "skip-roles", "[]", acc.skipRolesSet, &acc.skipRoles, extractOnSectionFieldFromMap) +} + +func (acc *importAccumulator) mergeSkipBots(fm map[string]any) { + mergeJSONStringListField(fm, "skip-bots", "[]", acc.skipBotsSet, &acc.skipBots, extractOnSectionFieldFromMap) +} + +func (acc *importAccumulator) mergeAmbientFolders(fm map[string]any) { + mergeJSONStringListField(fm, "ambient-folders", "[]", acc.ambientFoldersSet, &acc.ambientFolders, func(m map[string]any, field string) (string, error) { + return extractFieldJSONFromMap(m, field, "[]") + }) +} + +func (acc *importAccumulator) extractActivationSkipMatchFields(fm map[string]any, fullPath string) { + if acc.skipIfMatch == "" { + if skipJSON, skipErr := extractOnSectionAnyFieldFromMap(fm, "skip-if-match"); skipErr == nil && skipJSON != "" && skipJSON != "null" { + acc.skipIfMatch = skipJSON + parserLog.Printf("Extracted on.skip-if-match from import: %s", fullPath) + } + } + if acc.skipIfNoMatch == "" { + if skipJSON, skipErr := extractOnSectionAnyFieldFromMap(fm, "skip-if-no-match"); skipErr == nil && skipJSON != "" && skipJSON != "null" { + acc.skipIfNoMatch = skipJSON + parserLog.Printf("Extracted on.skip-if-no-match from import: %s", fullPath) + } + } +} + +func (acc *importAccumulator) extractActivationGitHubToken(fm map[string]any, fullPath string) { + if acc.activationGitHubToken != "" { + return + } + tokenJSON, tokenErr := extractOnSectionAnyFieldFromMap(fm, "github-token") + if tokenErr != nil || tokenJSON == "" || tokenJSON == "null" { + return + } + var token string + if jsonErr := json.Unmarshal([]byte(tokenJSON), &token); jsonErr == nil && token != "" { + acc.activationGitHubToken = token + parserLog.Printf("Extracted on.github-token from import: %s", fullPath) + } +} + +func (acc *importAccumulator) extractActivationGitHubAppFields(fm map[string]any, fullPath string) { + if acc.activationGitHubApp == "" { + if appJSON, appErr := extractOnSectionAnyFieldFromMap(fm, "github-app"); appErr == nil { + if validated := validateGitHubAppJSON(appJSON); validated != "" { + acc.activationGitHubApp = validated + parserLog.Printf("Extracted on.github-app from import: %s", fullPath) + } + } + } + if acc.topLevelGitHubApp == "" { + if appJSON, appErr := extractFieldJSONFromMap(fm, "github-app", ""); appErr == nil { + if validated := validateGitHubAppJSON(appJSON); validated != "" { + acc.topLevelGitHubApp = validated + parserLog.Printf("Extracted top-level github-app from import: %s", fullPath) + } + } + } +} + +func (acc *importAccumulator) extractCheckoutField(fm map[string]any, fullPath string) { + checkoutJSON, checkoutErr := extractFieldJSONFromMap(fm, "checkout", "") + if checkoutErr != nil || checkoutJSON == "" || checkoutJSON == "null" || checkoutJSON == "false" { + return + } + acc.checkouts = append(acc.checkouts, checkoutJSON) + parserLog.Printf("Extracted checkout from import: %s", fullPath) +} diff --git a/pkg/parser/import_field_extractor_engine.go b/pkg/parser/import_field_extractor_engine.go new file mode 100644 index 00000000000..1190f073272 --- /dev/null +++ b/pkg/parser/import_field_extractor_engine.go @@ -0,0 +1,241 @@ +// Package parser provides functions for parsing and processing workflow markdown files. +// import_field_extractor_engine.go implements extraction of engine and scalar/builder +// configuration fields (engine, max-turns, mcp-servers, safe-outputs, steps, runtimes, +// services, network, sandbox, permissions, secret-masking) from imported frontmatter. +package parser + +import ( + "encoding/json" + "strings" +) + +// extractEngineConfig extracts engine-related settings from the imported frontmatter map +// and accumulates them. Engine configs with only `mcp` sub-keys (no `id` or `runtime`) +// are not counted as engine specifications — they carry MCP gateway settings only. +// +// Side effects: acc.engines, acc.mergedEngineMCPToolTimeout, +// acc.mergedEngineMCPSessionTimeout, acc.mergedEngineModel. +func (acc *importAccumulator) extractEngineConfig(fm map[string]any, fullPath string) { + if modelStr, ok := fm["model"].(string); ok && modelStr != "" && acc.mergedEngineModel == "" { + acc.mergedEngineModel = modelStr + parserLog.Printf("Extracted top-level model preference from import %s: %s", fullPath, modelStr) + } + + engineVal, hasEngine := fm["engine"] + if !hasEngine { + return + } + parserLog.Printf("Found engine config in import: %s", fullPath) + + switch v := engineVal.(type) { + case string: + // String engine (e.g. "copilot") — always counts as an engine spec. + if engineJSON, merr := json.Marshal(v); merr == nil { + acc.engines = append(acc.engines, string(engineJSON)) + } + case map[string]any: + // Object engine — extract engine.mcp.* settings first, then decide + // whether to add to engines based on whether an engine ID is present. + if mcpVal, hasMCP := v["mcp"]; hasMCP { + acc.extractEngineMCPSettings(mcpVal, fullPath) + } + // Only add to engines list if this config specifies an actual engine + // (i.e. it carries an 'id' or 'runtime' field). Configs with only + // 'model' or 'mcp' settings are preferences, not engine selections, + // and must not trigger the "multiple engine fields" validation error. + _, hasID := v["id"] + _, hasRuntime := v["runtime"] + if hasID || hasRuntime { + if engineJSON, merr := json.Marshal(v); merr == nil { + acc.engines = append(acc.engines, string(engineJSON)) + } + } else { + // No engine ID or runtime — this is a model/MCP-only preference. + // Extract the model hint (first-wins) so it can be applied to the + // resolved engine after all imports are processed. + if modelStr, ok := v["model"].(string); ok && modelStr != "" { + if acc.mergedEngineModel == "" { + acc.mergedEngineModel = modelStr + parserLog.Printf("Extracted engine.model preference from import %s: %s", fullPath, modelStr) + } + } + } + default: + // Unexpected type — marshal and add to preserve existing behavior. + if engineJSON, merr := json.Marshal(engineVal); merr == nil { + acc.engines = append(acc.engines, string(engineJSON)) + } + } +} + +// extractEngineMCPSettings extracts engine.mcp.tool-timeout and engine.mcp.session-timeout +// from mcpVal (first-wins across all imports). +func (acc *importAccumulator) extractEngineMCPSettings(mcpVal any, fullPath string) { + mcpMap, ok := mcpVal.(map[string]any) + if !ok { + return + } + if acc.mergedEngineMCPToolTimeout == "" { + if ttStr, ok := mcpMap["tool-timeout"].(string); ok && ttStr != "" { + acc.mergedEngineMCPToolTimeout = ttStr + parserLog.Printf("Extracted engine.mcp.tool-timeout from import %s: %s", fullPath, ttStr) + } + } + if acc.mergedEngineMCPSessionTimeout == "" { + if stStr, ok := mcpMap["session-timeout"].(string); ok && stStr != "" { + acc.mergedEngineMCPSessionTimeout = stStr + parserLog.Printf("Extracted engine.mcp.session-timeout from import %s: %s", fullPath, stStr) + } + } +} + +// extractConfigFields extracts scalar and builder-based configuration fields from the +// frontmatter map and writes them into the appropriate accumulator builders and slices. +// +// Side effects: acc.mergedMaxTurns, acc.mergedMaxToolDenials, acc.mergedMaxRuns, acc.mergedMaxAICredits, +// acc.mergedMaxDailyAICredits, acc.mcpServersBuilder, +// acc.safeOutputs, acc.mcpScripts, acc.stepsBuilder, acc.runtimesBuilder, +// acc.servicesBuilder, acc.networkBuilder, acc.permissionsBuilder, +// acc.secretMaskingBuilder. +func (acc *importAccumulator) extractConfigFields(fm map[string]any, fullPath string) { + acc.extractFirstWinsJSONField(fm, fullPath, "max-turns", &acc.mergedMaxTurns) + acc.extractFirstWinsJSONField(fm, fullPath, "max-tool-denials", &acc.mergedMaxToolDenials) + acc.extractFirstWinsJSONField(fm, fullPath, "max-runs", &acc.mergedMaxRuns) + acc.extractFirstWinsJSONField(fm, fullPath, "max-turn-cache-misses", &acc.mergedMaxTurnCacheMisses) + acc.extractFirstWinsJSONField(fm, fullPath, "max-ai-credits", &acc.mergedMaxAICredits) + acc.extractFirstWinsJSONField(fm, fullPath, "max-daily-ai-credits", &acc.mergedMaxDailyAICredits) + + acc.appendJSONBuilderField(fm, "mcp-servers", "{}", &acc.mcpServersBuilder) + acc.plugins = append(acc.plugins, parseStringSliceField(fm["plugins"], false)...) + acc.appendJSONSliceField(fm, "safe-outputs", "{}", &acc.safeOutputs) + acc.appendJSONSliceField(fm, "mcp-scripts", "{}", &acc.mcpScripts) + acc.appendYAMLBuilderField(fm, "steps", &acc.stepsBuilder) + acc.appendJSONBuilderField(fm, "runtimes", "{}", &acc.runtimesBuilder) + acc.appendYAMLBuilderField(fm, "services", &acc.servicesBuilder) + acc.appendJSONBuilderField(fm, "network", "{}", &acc.networkBuilder) + acc.mergeSandboxAgentMounts(fm) + acc.mergeSandboxAgentRuntimeInstall(fm) + acc.appendJSONBuilderField(fm, "permissions", "{}", &acc.permissionsBuilder) + acc.appendJSONBuilderField(fm, "secret-masking", "{}", &acc.secretMaskingBuilder) +} + +func (acc *importAccumulator) mergeSandboxAgentMounts(fm map[string]any) { + sandboxVal, hasSandbox := fm["sandbox"] + if !hasSandbox { + return + } + + sandboxMap, ok := sandboxVal.(map[string]any) + if !ok { + return + } + + agentVal, hasAgent := sandboxMap["agent"] + if !hasAgent { + return + } + + agentMap, ok := agentVal.(map[string]any) + if !ok { + return + } + + mountsVal, hasMounts := agentMap["mounts"] + if !hasMounts { + return + } + + mounts, ok := mountsVal.([]any) + if !ok { + return + } + + for _, mountVal := range mounts { + mount, ok := mountVal.(string) + if !ok || mount == "" { + continue + } + if !acc.sandboxAgentMountsSet[mount] { + acc.sandboxAgentMountsSet[mount] = true + acc.sandboxAgentMounts = append(acc.sandboxAgentMounts, mount) + } + } +} + +// mergeSandboxAgentRuntimeInstall extracts sandbox.agent.runtime-install from an +// imported workflow's frontmatter. False wins: if any import sets runtime-install +// to false the accumulated value becomes false and stays false. +func (acc *importAccumulator) mergeSandboxAgentRuntimeInstall(fm map[string]any) { + // Already locked to false — no need to inspect further imports. + if acc.sandboxAgentRuntimeInstall != nil && !*acc.sandboxAgentRuntimeInstall { + return + } + + sandboxVal, hasSandbox := fm["sandbox"] + if !hasSandbox { + return + } + sandboxMap, ok := sandboxVal.(map[string]any) + if !ok { + return + } + agentVal, hasAgent := sandboxMap["agent"] + if !hasAgent { + return + } + agentMap, ok := agentVal.(map[string]any) + if !ok { + return + } + riVal, hasRI := agentMap["runtime-install"] + if !hasRI { + return + } + ri, ok := riVal.(bool) + if !ok { + return + } + if !ri { + f := false + acc.sandboxAgentRuntimeInstall = &f + } else if acc.sandboxAgentRuntimeInstall == nil { + t := true + acc.sandboxAgentRuntimeInstall = &t + } +} + +func (acc *importAccumulator) extractFirstWinsJSONField(fm map[string]any, fullPath, field string, target *string) { + if *target != "" { + return + } + fieldJSON, err := extractFieldJSONFromMap(fm, field, "") + if err != nil || fieldJSON == "" || fieldJSON == "null" { + return + } + *target = fieldJSON + parserLog.Printf("Extracted %s from import: %s", field, fullPath) +} + +func (acc *importAccumulator) appendJSONBuilderField(fm map[string]any, field, emptyValue string, builder *strings.Builder) { + content, err := extractFieldJSONFromMap(fm, field, emptyValue) + if err != nil || content == "" || content == emptyValue { + return + } + builder.WriteString(content + "\n") +} + +func (acc *importAccumulator) appendJSONSliceField(fm map[string]any, field, emptyValue string, target *[]string) { + content, err := extractFieldJSONFromMap(fm, field, emptyValue) + if err != nil || content == "" || content == emptyValue { + return + } + *target = append(*target, content) +} + +func (acc *importAccumulator) appendYAMLBuilderField(fm map[string]any, field string, builder *strings.Builder) { + content, err := extractYAMLFieldFromMap(fm, field) + if err != nil || content == "" { + return + } + builder.WriteString(content + "\n") +} diff --git a/pkg/parser/import_field_extractor_helpers.go b/pkg/parser/import_field_extractor_helpers.go new file mode 100644 index 00000000000..4138fa7f5d7 --- /dev/null +++ b/pkg/parser/import_field_extractor_helpers.go @@ -0,0 +1,49 @@ +// Package parser provides functions for parsing and processing workflow markdown files. +// import_field_extractor_helpers.go provides shared helper utilities used across +// import-field extractor concern files. +package parser + +import "encoding/json" + +func mergeJSONStringListField( + fm map[string]any, + field, emptyValue string, + seen map[string]bool, + merged *[]string, + extractor func(map[string]any, string) (string, error), +) { + content, err := extractor(fm, field) + if err != nil || content == "" || content == emptyValue { + return + } + var imported []string + if jsonErr := json.Unmarshal([]byte(content), &imported); jsonErr != nil { + return + } + for _, value := range imported { + if !seen[value] { + seen[value] = true + *merged = append(*merged, value) + } + } +} + +func parseStringSliceField(value any, keepEmpty bool) []string { + values, ok := value.([]any) + if !ok { + return nil + } + result := make([]string, 0, len(values)) + for _, v := range values { + if s, ok := v.(string); ok { + if s == "" && !keepEmpty { + continue + } + result = append(result, s) + } + } + if len(result) == 0 { + return nil + } + return result +} diff --git a/pkg/parser/import_field_extractor_models.go b/pkg/parser/import_field_extractor_models.go new file mode 100644 index 00000000000..673903e81b9 --- /dev/null +++ b/pkg/parser/import_field_extractor_models.go @@ -0,0 +1,146 @@ +// Package parser provides functions for parsing and processing workflow markdown files. +// import_field_extractor_models.go implements extraction and normalization of the +// models field: aliases, allow/block policies, provider cost overlays, and +// default-ai-credits-pricing. +package parser + +import ( + "encoding/json" + "fmt" + "maps" +) + +func (acc *importAccumulator) appendModelsField(fm map[string]any, importPath string) { + modelsContent, err := extractFieldJSONFromMap(fm, "models", "{}") + if err != nil || modelsContent == "" || modelsContent == "{}" { + return + } + var rawModels map[string]any + if jsonErr := json.Unmarshal([]byte(modelsContent), &rawModels); jsonErr != nil { + acc.warnings = append(acc.warnings, fmt.Sprintf("import %q: models field is not a valid object; skipping invalid value", importPath)) + return + } + if modelPolicy := normalizeModelPolicies(rawModels, importPath, &acc.warnings); len(modelPolicy) > 0 { + acc.modelPolicies = append(acc.modelPolicies, modelPolicy) + parserLog.Printf("Extracted model policy from import: allowed=%d, blocked=%d", len(modelPolicy["allowed"]), len(modelPolicy["blocked"])) + } + if providers, hasProviders := rawModels["providers"]; hasProviders { + if providerMap, ok := sanitizeModelProvidersForCosts(providers, importPath, &acc.warnings); ok { + acc.modelCosts = append(acc.modelCosts, map[string]any{"providers": providerMap}) + parserLog.Printf("Extracted model costs from import: providers=%d", len(providerMap)) + } + } + if acc.defaultAiCreditsPricing == nil { + if defaultPricing, hasDefaultPricing := rawModels["default-ai-credits-pricing"]; hasDefaultPricing { + if pricingMap, ok := defaultPricing.(map[string]any); ok { + acc.defaultAiCreditsPricing = maps.Clone(pricingMap) + parserLog.Printf("Extracted default-ai-credits-pricing from import: %s", importPath) + } else { + acc.warnings = append(acc.warnings, fmt.Sprintf("import %q: models.default-ai-credits-pricing must be an object; skipping invalid value", importPath)) + } + } + } + + aliasModels := make(map[string]any, len(rawModels)) + for key, value := range rawModels { + // providers is reserved for model-cost overlays and should not be treated + // as an alias key, even when aliases and providers coexist. + if key == "providers" || key == "default-ai-credits-pricing" || isModelPolicyKey(key) { + continue + } + aliasModels[key] = value + } + if len(aliasModels) == 0 { + return + } + modelsMap := normalizeModelAliases(aliasModels) + if len(modelsMap) > 0 { + acc.models = append(acc.models, modelsMap) + parserLog.Printf("Extracted model aliases from import: %d entries", len(modelsMap)) + } +} + +func normalizeModelPolicies(rawModels map[string]any, importPath string, warnings *[]string) map[string][]string { + parse := func(key string) []string { + value, exists := rawModels[key] + if !exists { + return nil + } + return parseModelPolicyField(value, key, importPath, warnings) + } + allowed := parse(modelPolicyAllowedKey) + blocked := parse(modelPolicyBlockedKey) + if len(allowed) == 0 && len(blocked) == 0 { + return nil + } + return map[string][]string{ + modelPolicyAllowedKey: allowed, + modelPolicyBlockedKey: blocked, + } +} + +func normalizeModelAliases(rawModels map[string]any) map[string][]string { + modelsMap := make(map[string][]string, len(rawModels)) + for k, v := range rawModels { + strs := parseStringSliceField(v, true) + if len(strs) == 0 { + continue + } + modelsMap[k] = strs + } + return modelsMap +} + +// parseModelPolicyField parses one imported models policy field as a string list. +// Invalid field shapes or entries are ignored and appended to warnings. +func parseModelPolicyField(value any, fieldName, importPath string, warnings *[]string) []string { + values, ok := value.([]any) + if !ok { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s must be an array; skipping invalid value", importPath, fieldName)) + return nil + } + result := make([]string, 0, len(values)) + for _, v := range values { + s, ok := v.(string) + if !ok { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s contains a non-string entry; skipping invalid entry", importPath, fieldName)) + continue + } + if s == "" { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.%s contains an empty string entry; skipping invalid entry", importPath, fieldName)) + continue + } + result = append(result, s) + } + if len(result) == 0 { + return nil + } + return result +} + +// sanitizeModelProvidersForCosts validates models.providers from an import. +// It returns the provider map and true when the input is a non-empty object; otherwise false. +func sanitizeModelProvidersForCosts(providers any, importPath string, warnings *[]string) (map[string]any, bool) { + providerMap, ok := providers.(map[string]any) + if !ok || len(providerMap) == 0 { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers must be a non-empty object; skipping invalid value", importPath)) + return nil, false + } + sanitizedProviders := make(map[string]any, len(providerMap)) + for providerName, providerValue := range providerMap { + if isModelPolicyKey(providerName) || providerName == "blocked" { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers.%s is reserved for policy and ignored in cost data", importPath, providerName)) + continue + } + sanitizedProviders[providerName] = providerValue + } + if len(sanitizedProviders) == 0 { + *warnings = append(*warnings, fmt.Sprintf("import %q: models.providers must contain at least one non-policy provider key", importPath)) + return nil, false + } + return sanitizedProviders, true +} + +func isModelPolicyKey(key string) bool { + return key == modelPolicyAllowedKey || key == modelPolicyBlockedKey +} diff --git a/pkg/parser/import_field_extractor_steps.go b/pkg/parser/import_field_extractor_steps.go new file mode 100644 index 00000000000..451c24d76cd --- /dev/null +++ b/pkg/parser/import_field_extractor_steps.go @@ -0,0 +1,146 @@ +// Package parser provides functions for parsing and processing workflow markdown files. +// import_field_extractor_steps.go implements extraction of step, job, environment, +// label, cache, feature-flag, run-install-scripts, and observability fields from +// imported frontmatter. +package parser + +import ( + "encoding/json" + "fmt" +) + +// extractStepAndJobFields extracts step and job configuration fields from the frontmatter +// map. Environment variable conflict detection is performed: if the same env var is +// defined in two different imports, an error is returned. +// +// Side effects: acc.preStepsBuilder, acc.preAgentStepsBuilder, acc.postStepsBuilder, +// acc.jobsBuilder, acc.envBuilder, acc.envSources. +func (acc *importAccumulator) extractStepAndJobFields(fm map[string]any, importPath string) error { + // Extract pre-steps (prepend in order). + if preStepsContent, err := extractYAMLFieldFromMap(fm, "pre-steps"); err == nil && preStepsContent != "" { + acc.preStepsBuilder.WriteString(preStepsContent + "\n") + } + + // Extract pre-agent-steps (prepend in order). + if preAgentStepsContent, err := extractYAMLFieldFromMap(fm, "pre-agent-steps"); err == nil && preAgentStepsContent != "" { + acc.preAgentStepsBuilder.WriteString(preAgentStepsContent + "\n") + } + + // Extract post-steps (append in order). + if postStepsContent, err := extractYAMLFieldFromMap(fm, "post-steps"); err == nil && postStepsContent != "" { + acc.postStepsBuilder.WriteString(postStepsContent + "\n") + } + + // Extract jobs (append in order; merged into custom jobs map). + if jobsContent, err := extractFieldJSONFromMap(fm, "jobs", "{}"); err == nil && jobsContent != "" && jobsContent != "{}" { + acc.jobsBuilder.WriteString(jobsContent + "\n") + } + + // Extract env (append in order; main workflow env takes precedence). + // Conflicts between two imports are disallowed — only the main workflow may override imported vars. + envContent, err := extractFieldJSONFromMap(fm, "env", "{}") + if err == nil && envContent != "" && envContent != "{}" { + var envMap map[string]any + if jsonErr := json.Unmarshal([]byte(envContent), &envMap); jsonErr == nil { + for key := range envMap { + if existingSource, exists := acc.envSources[key]; exists { + return fmt.Errorf("env variable %q is defined in multiple imports: %q and %q; remove the duplicate definition from one of the imports, or move it to the main workflow to override imported values", key, existingSource, importPath) + } + acc.envSources[key] = importPath + } + acc.envBuilder.WriteString(envContent + "\n") + } + } + + return nil +} + +// extractFeatureAndObservabilityFields extracts labels, cache, feature flags, model +// aliases, the run-install-scripts flag, observability configuration, and excluded-env +// from the frontmatter map. +// +// Side effects: acc.labels, acc.labelsSet, acc.caches, acc.features, acc.models, +// acc.runInstallScripts, acc.observabilityConfigs, acc.excludedEnv, acc.excludedEnvSet. +func (acc *importAccumulator) extractFeatureAndObservabilityFields(fm map[string]any, fullPath string) { + acc.mergeLabels(fm) + acc.appendCacheField(fm) + acc.appendFeaturesField(fm) + acc.appendModelsField(fm, fullPath) + acc.extractRunInstallScripts(fm, fullPath) + acc.appendObservabilityField(fm, fullPath) + acc.mergeExcludedEnv(fm) +} + +func (acc *importAccumulator) mergeExcludedEnv(fm map[string]any) { + mergeJSONStringListField(fm, "excluded-env", "[]", acc.excludedEnvSet, &acc.excludedEnv, func(m map[string]any, field string) (string, error) { + return extractFieldJSONFromMap(m, field, "[]") + }) +} + +func (acc *importAccumulator) mergeLabels(fm map[string]any) { + mergeJSONStringListField(fm, "labels", "[]", acc.labelsSet, &acc.labels, func(m map[string]any, field string) (string, error) { + return extractFieldJSONFromMap(m, field, "[]") + }) +} + +func (acc *importAccumulator) appendCacheField(fm map[string]any) { + if cacheContent, err := extractFieldJSONFromMap(fm, "cache", "{}"); err == nil && cacheContent != "" && cacheContent != "{}" { + acc.caches = append(acc.caches, cacheContent) + } +} + +func (acc *importAccumulator) appendFeaturesField(fm map[string]any) { + featuresContent, err := extractFieldJSONFromMap(fm, "features", "{}") + if err != nil || featuresContent == "" || featuresContent == "{}" { + return + } + var featuresMap map[string]any + if jsonErr := json.Unmarshal([]byte(featuresContent), &featuresMap); jsonErr == nil { + acc.features = append(acc.features, featuresMap) + parserLog.Printf("Extracted features from import: %d entries", len(featuresMap)) + } +} + +func (acc *importAccumulator) extractRunInstallScripts(fm map[string]any, fullPath string) { + if acc.runInstallScripts { + return + } + if hasNodeRuntimeRunInstallScripts(fm) { + acc.runInstallScripts = true + parserLog.Printf("Extracted runtimes.node.run-install-scripts: true from import: %s", fullPath) + } +} + +func hasNodeRuntimeRunInstallScripts(fm map[string]any) bool { + runtimesAny, hasRuntimes := fm["runtimes"] + if !hasRuntimes { + return false + } + runtimesMap, ok := runtimesAny.(map[string]any) + if !ok { + return false + } + nodeAny, hasNode := runtimesMap["node"] + if !hasNode { + return false + } + nodeMap, ok := nodeAny.(map[string]any) + if !ok { + return false + } + rsAny, hasRS := nodeMap["run-install-scripts"] + if !hasRS { + return false + } + rsBool, ok := rsAny.(bool) + return ok && rsBool +} + +func (acc *importAccumulator) appendObservabilityField(fm map[string]any, fullPath string) { + obsContent, obsErr := extractFieldJSONFromMap(fm, "observability", "{}") + if obsErr != nil || obsContent == "" || obsContent == "{}" { + return + } + acc.observabilityConfigs = append(acc.observabilityConfigs, obsContent) + parserLog.Printf("Extracted observability from import: %s", fullPath) +}