From 3b79238c21883a01c5c2ad10b7ec8e2dc84146b6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 11:18:46 +0000 Subject: [PATCH 1/6] Initial plan From fcf94018ee84b59731eba47b0b2abed0470828bc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 11:33:01 +0000 Subject: [PATCH 2/6] Add --firewall flag to logs command for filtering by firewall usage Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs.go | 44 ++++- pkg/cli/logs_firewall_filter_test.go | 235 +++++++++++++++++++++++++++ pkg/cli/logs_test.go | 4 +- 3 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 pkg/cli/logs_firewall_filter_test.go diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index 6e321266edf..4f86e3fb5b1 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -327,6 +327,8 @@ Examples: ` + constants.CLIExtensionPrefix + ` logs --engine claude # Filter logs by claude engine ` + constants.CLIExtensionPrefix + ` logs --engine codex # Filter logs by codex engine ` + constants.CLIExtensionPrefix + ` logs --engine copilot # Filter logs by copilot engine + ` + constants.CLIExtensionPrefix + ` logs --firewall true # Filter logs with firewall enabled + ` + constants.CLIExtensionPrefix + ` logs --firewall false # Filter logs without firewall ` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory ` + constants.CLIExtensionPrefix + ` logs --branch main # Filter logs by branch name ` + constants.CLIExtensionPrefix + ` logs --branch feature-xyz # Filter logs by feature branch @@ -371,6 +373,7 @@ Examples: verbose, _ := cmd.Flags().GetBool("verbose") toolGraph, _ := cmd.Flags().GetBool("tool-graph") noStaged, _ := cmd.Flags().GetBool("no-staged") + firewallFilter, _ := cmd.Flags().GetString("firewall") parse, _ := cmd.Flags().GetBool("parse") jsonOutput, _ := cmd.Flags().GetBool("json") timeout, _ := cmd.Flags().GetInt("timeout") @@ -413,7 +416,16 @@ Examples: } } - if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, branch, beforeRunID, afterRunID, verbose, toolGraph, noStaged, parse, jsonOutput, timeout); err != nil { + // Validate firewall parameter + if firewallFilter != "" && firewallFilter != "true" && firewallFilter != "false" { + fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ + Type: "error", + Message: fmt.Sprintf("invalid firewall value '%s'. Must be 'true', 'false', or empty", firewallFilter), + })) + os.Exit(1) + } + + if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, branch, beforeRunID, afterRunID, verbose, toolGraph, noStaged, firewallFilter, parse, jsonOutput, timeout); err != nil { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", Message: err.Error(), @@ -434,6 +446,7 @@ Examples: logsCmd.Flags().Int64("after-run-id", 0, "Filter runs with database ID after this value (exclusive)") logsCmd.Flags().Bool("tool-graph", false, "Generate Mermaid tool sequence graph from agent logs") logsCmd.Flags().Bool("no-staged", false, "Filter out staged workflow runs (exclude runs with staged: true in aw_info.json)") + logsCmd.Flags().String("firewall", "", "Filter runs by firewall usage: 'true' (only runs with firewall), 'false' (only runs without firewall), or '' (all runs)") logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing markdown to log.md and firewall.md") logsCmd.Flags().Bool("json", false, "Output logs data as JSON instead of formatted console tables") logsCmd.Flags().Int("timeout", 0, "Maximum time in seconds to spend downloading logs (0 = no timeout)") @@ -442,7 +455,7 @@ Examples: } // DownloadWorkflowLogs downloads and analyzes workflow logs with metrics -func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine, branch string, beforeRunID, afterRunID int64, verbose bool, toolGraph bool, noStaged bool, parse bool, jsonOutput bool, timeout int) error { +func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine, branch string, beforeRunID, afterRunID int64, verbose bool, toolGraph bool, noStaged bool, firewallFilter string, parse bool, jsonOutput bool, timeout int) error { logsLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s", workflowName, count, startDate, endDate, outputDir) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Fetching workflow runs from GitHub Actions...")) @@ -615,6 +628,33 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou } } + // Apply firewall filtering if --firewall flag is specified + if firewallFilter != "" { + // Check the firewall field in aw_info.json + awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") + info, err := parseAwInfo(awInfoPath, verbose) + var hasFirewall bool + if err == nil && info != nil { + // Firewall is enabled if steps.firewall is non-empty (e.g., "squid") + hasFirewall = info.Steps.Firewall != "" + } + + // Check if the run matches the filter + filterRequiresFirewall := firewallFilter == "true" + if filterRequiresFirewall && !hasFirewall { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall true)", result.Run.DatabaseID))) + } + continue + } + if !filterRequiresFirewall && hasFirewall { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --firewall false)", result.Run.DatabaseID))) + } + continue + } + } + // Update run with metrics and path run := result.Run run.TokenUsage = result.Metrics.TokenUsage diff --git a/pkg/cli/logs_firewall_filter_test.go b/pkg/cli/logs_firewall_filter_test.go new file mode 100644 index 00000000000..598225c04b3 --- /dev/null +++ b/pkg/cli/logs_firewall_filter_test.go @@ -0,0 +1,235 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// TestParseAwInfo_FirewallField verifies that the firewall field is correctly parsed from aw_info.json +func TestParseAwInfo_FirewallField(t *testing.T) { + tests := []struct { + name string + jsonContent string + expectedFirewall string + description string + }{ + { + name: "firewall enabled with squid", + jsonContent: `{ + "engine_id": "copilot", + "engine_name": "Copilot", + "model": "gpt-4", + "version": "1.0", + "workflow_name": "test-workflow", + "staged": false, + "steps": { + "firewall": "squid" + }, + "created_at": "2025-01-27T15:00:00Z" + }`, + expectedFirewall: "squid", + description: "Should detect firewall enabled when steps.firewall is 'squid'", + }, + { + name: "firewall disabled (empty string)", + jsonContent: `{ + "engine_id": "copilot", + "engine_name": "Copilot", + "model": "gpt-4", + "version": "1.0", + "workflow_name": "test-workflow", + "staged": false, + "steps": { + "firewall": "" + }, + "created_at": "2025-01-27T15:00:00Z" + }`, + expectedFirewall: "", + description: "Should detect firewall disabled when steps.firewall is empty string", + }, + { + name: "no steps field (backward compatibility)", + jsonContent: `{ + "engine_id": "claude", + "engine_name": "Claude", + "model": "claude-3-sonnet", + "version": "20240620", + "workflow_name": "test-workflow", + "staged": false, + "created_at": "2025-01-27T15:00:00Z" + }`, + expectedFirewall: "", + description: "Should handle missing steps field (backward compatibility)", + }, + { + name: "steps field without firewall", + jsonContent: `{ + "engine_id": "copilot", + "engine_name": "Copilot", + "model": "gpt-4", + "version": "1.0", + "workflow_name": "test-workflow", + "staged": false, + "steps": {}, + "created_at": "2025-01-27T15:00:00Z" + }`, + expectedFirewall: "", + description: "Should handle steps field without firewall subfield", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary file with the JSON content + tempDir := t.TempDir() + awInfoPath := filepath.Join(tempDir, "aw_info.json") + + err := os.WriteFile(awInfoPath, []byte(tt.jsonContent), 0644) + if err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + // Parse the aw_info.json file + info, err := parseAwInfo(awInfoPath, false) + if err != nil { + t.Fatalf("Failed to parse aw_info.json: %v", err) + } + + // Check the firewall field + if info.Steps.Firewall != tt.expectedFirewall { + t.Errorf("%s\nExpected firewall: '%s', got: '%s'", + tt.description, tt.expectedFirewall, info.Steps.Firewall) + } + + t.Logf("✓ %s", tt.description) + }) + } +} + +// TestFirewallFilterLogic verifies the filtering logic for firewall parameter +func TestFirewallFilterLogic(t *testing.T) { + tests := []struct { + name string + firewallInJSON string + filterValue string + shouldBeSkipped bool + description string + }{ + { + name: "filter=true, has firewall - should NOT skip", + firewallInJSON: "squid", + filterValue: "true", + shouldBeSkipped: false, + description: "Run with firewall should pass when filter='true'", + }, + { + name: "filter=true, no firewall - should skip", + firewallInJSON: "", + filterValue: "true", + shouldBeSkipped: true, + description: "Run without firewall should be skipped when filter='true'", + }, + { + name: "filter=false, has firewall - should skip", + firewallInJSON: "squid", + filterValue: "false", + shouldBeSkipped: true, + description: "Run with firewall should be skipped when filter='false'", + }, + { + name: "filter=false, no firewall - should NOT skip", + firewallInJSON: "", + filterValue: "false", + shouldBeSkipped: false, + description: "Run without firewall should pass when filter='false'", + }, + { + name: "filter empty, has firewall - should NOT skip", + firewallInJSON: "squid", + filterValue: "", + shouldBeSkipped: false, + description: "Run with firewall should pass when no filter specified", + }, + { + name: "filter empty, no firewall - should NOT skip", + firewallInJSON: "", + filterValue: "", + shouldBeSkipped: false, + description: "Run without firewall should pass when no filter specified", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the filtering logic from DownloadWorkflowLogs + var hasFirewall bool + if tt.firewallInJSON != "" { + hasFirewall = true + } + + var shouldSkip bool + if tt.filterValue != "" { + filterRequiresFirewall := tt.filterValue == "true" + if filterRequiresFirewall && !hasFirewall { + shouldSkip = true + } + if !filterRequiresFirewall && hasFirewall { + shouldSkip = true + } + } + + if shouldSkip != tt.shouldBeSkipped { + t.Errorf("%s\nExpected shouldSkip=%v, got shouldSkip=%v", + tt.description, tt.shouldBeSkipped, shouldSkip) + } + + t.Logf("✓ %s (shouldSkip=%v)", tt.description, shouldSkip) + }) + } +} + +// TestAwInfoWithFirewallMarshaling verifies that AwInfo with firewall field marshals correctly +func TestAwInfoWithFirewallMarshaling(t *testing.T) { + info := AwInfo{ + EngineID: "copilot", + EngineName: "Copilot", + Model: "gpt-4", + Version: "1.0", + WorkflowName: "test-workflow", + Staged: false, + Steps: AwInfoSteps{ + Firewall: "squid", + }, + CreatedAt: "2025-01-27T15:00:00Z", + } + + jsonData, err := json.Marshal(info) + if err != nil { + t.Fatalf("Failed to marshal AwInfo: %v", err) + } + + // Verify that the JSON contains the steps.firewall field + var result map[string]interface{} + err = json.Unmarshal(jsonData, &result) + if err != nil { + t.Fatalf("Failed to unmarshal marshaled JSON: %v", err) + } + + steps, ok := result["steps"].(map[string]interface{}) + if !ok { + t.Fatal("Expected 'steps' field in marshaled JSON") + } + + firewall, ok := steps["firewall"].(string) + if !ok { + t.Fatal("Expected 'firewall' field in steps object") + } + + if firewall != "squid" { + t.Errorf("Expected firewall to be 'squid', got: '%s'", firewall) + } + + t.Log("✓ AwInfo with firewall field marshals correctly") +} diff --git a/pkg/cli/logs_test.go b/pkg/cli/logs_test.go index d3f3c144748..9e3756cddf3 100644 --- a/pkg/cli/logs_test.go +++ b/pkg/cli/logs_test.go @@ -19,7 +19,7 @@ func TestDownloadWorkflowLogs(t *testing.T) { // Test the DownloadWorkflowLogs function // This should either fail with auth error (if not authenticated) // or succeed with no results (if authenticated but no workflows match) - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", "", 0, 0, false, false, false, false, false, 0) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", "", 0, 0, false, false, false, "", false, false, 0) // If GitHub CLI is authenticated, the function may succeed but find no results // If not authenticated, it should return an auth error @@ -915,7 +915,7 @@ func TestDownloadWorkflowLogsWithEngineFilter(t *testing.T) { if !tt.expectError { // For valid engines, test that the function can be called without panic // It may still fail with auth errors, which is expected - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, "", 0, 0, false, false, false, false, false, 0) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, "", 0, 0, false, false, false, "", false, false, 0) // Clean up any created directories os.RemoveAll("./test-logs") From 695cd6ec455337e2d906cef4a80791bb0f83610b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 11:35:06 +0000 Subject: [PATCH 3/6] Update daily-firewall-report workflow to use --firewall flag Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/daily-firewall-report.lock.yml | 33 +++++++++++-------- .github/workflows/daily-firewall-report.md | 33 +++++++++++-------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index 897f9262b5a..f47fab723b7 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1768,26 +1768,31 @@ jobs: This prevents unnecessary re-analysis of the same data and significantly reduces token usage. - ### Step 1: Identify Workflows with Firewall Feature + ### Step 1: Collect Recent Firewall-Enabled Workflow Runs - 1. List all workflows in the repository - 2. For each workflow that has `network.firewall: true` in its frontmatter, note the workflow name - 3. Create a list of all firewall-enabled workflows + Use the `logs` command with `--firewall true` to efficiently collect workflow runs that have firewall enabled: - **Example frontmatter structure:** - ```yaml - network: - firewall: true + ```bash + # Get up to 100 recent workflow runs that used firewall (within past 7 days) + gh aw logs --firewall true --start-date -7d -c 100 ``` - **Note:** The firewall field is under `network`, not `features`. + This command: + 1. Filters runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) + 2. Returns only runs where firewall was enabled + 3. Limits to runs from the past 7 days + 4. Returns up to 100 matching runs + + **Alternative:** To get runs for all firewall-enabled workflows across all time: + ```bash + gh aw logs --firewall true -c 100 + ``` - ### Step 2: Collect Recent Workflow Runs + ### Step 2: Analyze Firewall Logs from Collected Runs - For each firewall-enabled workflow: - 1. Get up to 10 workflow runs that occurred within the past 7 days (if there are fewer than 10 runs in that window, include all available; if there are more, include only the most recent 10) - 2. For each run ID, use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information - 3. Store the run ID, workflow name, and timestamp for tracking + For each run collected in Step 1: + 1. Use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information + 2. Store the run ID, workflow name, and timestamp for tracking **Using the audit tool:** ```bash diff --git a/.github/workflows/daily-firewall-report.md b/.github/workflows/daily-firewall-report.md index ba9d0fe04a0..b31c13e6807 100644 --- a/.github/workflows/daily-firewall-report.md +++ b/.github/workflows/daily-firewall-report.md @@ -161,26 +161,31 @@ Generate a comprehensive daily report of all rejected domains across all agentic This prevents unnecessary re-analysis of the same data and significantly reduces token usage. -### Step 1: Identify Workflows with Firewall Feature +### Step 1: Collect Recent Firewall-Enabled Workflow Runs -1. List all workflows in the repository -2. For each workflow that has `network.firewall: true` in its frontmatter, note the workflow name -3. Create a list of all firewall-enabled workflows +Use the `logs` command with `--firewall true` to efficiently collect workflow runs that have firewall enabled: -**Example frontmatter structure:** -```yaml -network: - firewall: true +```bash +# Get up to 100 recent workflow runs that used firewall (within past 7 days) +gh aw logs --firewall true --start-date -7d -c 100 ``` -**Note:** The firewall field is under `network`, not `features`. +This command: +1. Filters runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) +2. Returns only runs where firewall was enabled +3. Limits to runs from the past 7 days +4. Returns up to 100 matching runs + +**Alternative:** To get runs for all firewall-enabled workflows across all time: +```bash +gh aw logs --firewall true -c 100 +``` -### Step 2: Collect Recent Workflow Runs +### Step 2: Analyze Firewall Logs from Collected Runs -For each firewall-enabled workflow: -1. Get up to 10 workflow runs that occurred within the past 7 days (if there are fewer than 10 runs in that window, include all available; if there are more, include only the most recent 10) -2. For each run ID, use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information -3. Store the run ID, workflow name, and timestamp for tracking +For each run collected in Step 1: +1. Use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information +2. Store the run ID, workflow name, and timestamp for tracking **Using the audit tool:** ```bash From 716e21779dc5e2b0ae64b5ef6ccce7c157d095a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 11:48:50 +0000 Subject: [PATCH 4/6] Update --firewall flag to not allow empty value Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index 4f86e3fb5b1..c8888a815b1 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -420,7 +420,7 @@ Examples: if firewallFilter != "" && firewallFilter != "true" && firewallFilter != "false" { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", - Message: fmt.Sprintf("invalid firewall value '%s'. Must be 'true', 'false', or empty", firewallFilter), + Message: fmt.Sprintf("invalid firewall value '%s'. Must be 'true' or 'false'", firewallFilter), })) os.Exit(1) } @@ -446,7 +446,7 @@ Examples: logsCmd.Flags().Int64("after-run-id", 0, "Filter runs with database ID after this value (exclusive)") logsCmd.Flags().Bool("tool-graph", false, "Generate Mermaid tool sequence graph from agent logs") logsCmd.Flags().Bool("no-staged", false, "Filter out staged workflow runs (exclude runs with staged: true in aw_info.json)") - logsCmd.Flags().String("firewall", "", "Filter runs by firewall usage: 'true' (only runs with firewall), 'false' (only runs without firewall), or '' (all runs)") + logsCmd.Flags().String("firewall", "", "Filter runs by firewall usage: 'true' (only runs with firewall) or 'false' (only runs without firewall)") logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing markdown to log.md and firewall.md") logsCmd.Flags().Bool("json", false, "Output logs data as JSON instead of formatted console tables") logsCmd.Flags().Int("timeout", 0, "Maximum time in seconds to spend downloading logs (0 = no timeout)") From d0d37a9b0857d5dd305d5f0afaa49fd99bb3732c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 12:01:32 +0000 Subject: [PATCH 5/6] Add firewall parameter to logs tool in MCP server and update daily-firewall-report workflow Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/daily-firewall-report.lock.yml | 73 +++++++++++-------- .github/workflows/daily-firewall-report.md | 73 +++++++++++-------- pkg/cli/mcp_server.go | 4 + 3 files changed, 86 insertions(+), 64 deletions(-) diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index f47fab723b7..e4c79cfee9b 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1770,52 +1770,61 @@ jobs: ### Step 1: Collect Recent Firewall-Enabled Workflow Runs - Use the `logs` command with `--firewall true` to efficiently collect workflow runs that have firewall enabled: - - ```bash - # Get up to 100 recent workflow runs that used firewall (within past 7 days) - gh aw logs --firewall true --start-date -7d -c 100 - ``` - - This command: - 1. Filters runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) - 2. Returns only runs where firewall was enabled - 3. Limits to runs from the past 7 days - 4. Returns up to 100 matching runs - - **Alternative:** To get runs for all firewall-enabled workflows across all time: - ```bash - gh aw logs --firewall true -c 100 + Use the `logs` tool from the agentic-workflows MCP server to efficiently collect workflow runs that have firewall enabled: + + **Using the logs tool:** + Call the `logs` tool with the following parameters: + - `firewall`: "true" (to filter only runs with firewall enabled) + - `start_date`: "-7d" (to get runs from the past 7 days) + - `count`: 100 (to get up to 100 matching runs) + + The tool will: + 1. Filter runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) + 2. Return only runs where firewall was enabled + 3. Limit to runs from the past 7 days + 4. Return up to 100 matching runs + + **Tool call example:** + ```json + { + "firewall": "true", + "start_date": "-7d", + "count": 100 + } ``` ### Step 2: Analyze Firewall Logs from Collected Runs For each run collected in Step 1: - 1. Use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information + 1. Use the `audit` tool from the agentic-workflows MCP server to get detailed firewall information 2. Store the run ID, workflow name, and timestamp for tracking **Using the audit tool:** - ```bash - # Get firewall analysis in JSON format - gh aw audit --json - - # Example jq filter to extract firewall data: - gh aw audit --json | jq '{ - run_id: .overview.run_id, - workflow: .overview.workflow_name, - firewall: .firewall_analysis // {}, - denied_domains: .firewall_analysis.denied_domains // [], - allowed_domains: .firewall_analysis.allowed_domains // [], - total_requests: .firewall_analysis.total_requests // 0, - denied_requests: .firewall_analysis.denied_requests // 0 - }' + Call the `audit` tool with the run_id parameter for each run from Step 1. + + **Tool call example:** + ```json + { + "run_id": 12345678 + } ``` - **Important:** Do NOT manually download and parse firewall log files. Always use the `audit` tool which provides structured firewall analysis data including: + The audit tool returns structured firewall analysis data including: - Total requests, allowed requests, denied requests - Lists of allowed and denied domains - Request statistics per domain + **Example of extracting firewall data from audit result:** + ```javascript + // From the audit tool result, access: + result.firewall_analysis.denied_domains // Array of denied domain names + result.firewall_analysis.allowed_domains // Array of allowed domain names + result.firewall_analysis.total_requests // Total number of network requests + result.firewall_analysis.denied_requests // Number of denied requests + ``` + + **Important:** Do NOT manually download and parse firewall log files. Always use the `audit` tool which provides structured firewall analysis data. + ### Step 3: Parse and Analyze Firewall Logs Use the JSON output from the `audit` tool to extract firewall information. The `firewall_analysis` field in the audit JSON contains: diff --git a/.github/workflows/daily-firewall-report.md b/.github/workflows/daily-firewall-report.md index b31c13e6807..fa80f092c29 100644 --- a/.github/workflows/daily-firewall-report.md +++ b/.github/workflows/daily-firewall-report.md @@ -163,52 +163,61 @@ This prevents unnecessary re-analysis of the same data and significantly reduces ### Step 1: Collect Recent Firewall-Enabled Workflow Runs -Use the `logs` command with `--firewall true` to efficiently collect workflow runs that have firewall enabled: - -```bash -# Get up to 100 recent workflow runs that used firewall (within past 7 days) -gh aw logs --firewall true --start-date -7d -c 100 -``` - -This command: -1. Filters runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) -2. Returns only runs where firewall was enabled -3. Limits to runs from the past 7 days -4. Returns up to 100 matching runs - -**Alternative:** To get runs for all firewall-enabled workflows across all time: -```bash -gh aw logs --firewall true -c 100 +Use the `logs` tool from the agentic-workflows MCP server to efficiently collect workflow runs that have firewall enabled: + +**Using the logs tool:** +Call the `logs` tool with the following parameters: +- `firewall`: "true" (to filter only runs with firewall enabled) +- `start_date`: "-7d" (to get runs from the past 7 days) +- `count`: 100 (to get up to 100 matching runs) + +The tool will: +1. Filter runs based on the `steps.firewall` field in `aw_info.json` (e.g., "squid" when enabled) +2. Return only runs where firewall was enabled +3. Limit to runs from the past 7 days +4. Return up to 100 matching runs + +**Tool call example:** +```json +{ + "firewall": "true", + "start_date": "-7d", + "count": 100 +} ``` ### Step 2: Analyze Firewall Logs from Collected Runs For each run collected in Step 1: -1. Use the `audit` tool from the agentic-workflows MCP server with `--json` flag to get detailed firewall information +1. Use the `audit` tool from the agentic-workflows MCP server to get detailed firewall information 2. Store the run ID, workflow name, and timestamp for tracking **Using the audit tool:** -```bash -# Get firewall analysis in JSON format -gh aw audit --json - -# Example jq filter to extract firewall data: -gh aw audit --json | jq '{ - run_id: .overview.run_id, - workflow: .overview.workflow_name, - firewall: .firewall_analysis // {}, - denied_domains: .firewall_analysis.denied_domains // [], - allowed_domains: .firewall_analysis.allowed_domains // [], - total_requests: .firewall_analysis.total_requests // 0, - denied_requests: .firewall_analysis.denied_requests // 0 -}' +Call the `audit` tool with the run_id parameter for each run from Step 1. + +**Tool call example:** +```json +{ + "run_id": 12345678 +} ``` -**Important:** Do NOT manually download and parse firewall log files. Always use the `audit` tool which provides structured firewall analysis data including: +The audit tool returns structured firewall analysis data including: - Total requests, allowed requests, denied requests - Lists of allowed and denied domains - Request statistics per domain +**Example of extracting firewall data from audit result:** +```javascript +// From the audit tool result, access: +result.firewall_analysis.denied_domains // Array of denied domain names +result.firewall_analysis.allowed_domains // Array of allowed domain names +result.firewall_analysis.total_requests // Total number of network requests +result.firewall_analysis.denied_requests // Number of denied requests +``` + +**Important:** Do NOT manually download and parse firewall log files. Always use the `audit` tool which provides structured firewall analysis data. + ### Step 3: Parse and Analyze Firewall Logs Use the JSON output from the `audit` tool to extract firewall information. The `firewall_analysis` field in the audit JSON contains: diff --git a/pkg/cli/mcp_server.go b/pkg/cli/mcp_server.go index 69a5f361b00..262204142ec 100644 --- a/pkg/cli/mcp_server.go +++ b/pkg/cli/mcp_server.go @@ -243,6 +243,7 @@ Note: Output can be filtered using the jq parameter.`, StartDate string `json:"start_date,omitempty" jsonschema:"Filter runs created after this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)"` EndDate string `json:"end_date,omitempty" jsonschema:"Filter runs created before this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)"` Engine string `json:"engine,omitempty" jsonschema:"Filter logs by agentic engine type (claude, codex, copilot)"` + Firewall string `json:"firewall,omitempty" jsonschema:"Filter runs by firewall usage: 'true' (only runs with firewall) or 'false' (only runs without firewall)"` Branch string `json:"branch,omitempty" jsonschema:"Filter runs by branch name"` AfterRunID int64 `json:"after_run_id,omitempty" jsonschema:"Filter runs with database ID after this value (exclusive)"` BeforeRunID int64 `json:"before_run_id,omitempty" jsonschema:"Filter runs with database ID before this value (exclusive)"` @@ -286,6 +287,9 @@ to filter the output to a manageable size, or adjust the 'max_tokens' parameter. if args.Engine != "" { cmdArgs = append(cmdArgs, "--engine", args.Engine) } + if args.Firewall != "" { + cmdArgs = append(cmdArgs, "--firewall", args.Firewall) + } if args.Branch != "" { cmdArgs = append(cmdArgs, "--branch", args.Branch) } From 9a169bea6313fb5a59fa709c87e3b72fcd86fa09 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 12:21:49 +0000 Subject: [PATCH 6/6] Convert firewall CLI flag to boolean and optimize parseAwInfo calls Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/daily-firewall-report.lock.yml | 4 +- .github/workflows/daily-firewall-report.md | 4 +- pkg/cli/logs.go | 58 ++++++++++--------- pkg/cli/logs_test.go | 4 +- pkg/cli/mcp_server.go | 19 +++++- 5 files changed, 53 insertions(+), 36 deletions(-) diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index e4c79cfee9b..5545beea1ff 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1774,7 +1774,7 @@ jobs: **Using the logs tool:** Call the `logs` tool with the following parameters: - - `firewall`: "true" (to filter only runs with firewall enabled) + - `firewall`: true (boolean - to filter only runs with firewall enabled) - `start_date`: "-7d" (to get runs from the past 7 days) - `count`: 100 (to get up to 100 matching runs) @@ -1787,7 +1787,7 @@ jobs: **Tool call example:** ```json { - "firewall": "true", + "firewall": true, "start_date": "-7d", "count": 100 } diff --git a/.github/workflows/daily-firewall-report.md b/.github/workflows/daily-firewall-report.md index fa80f092c29..6a30536f1be 100644 --- a/.github/workflows/daily-firewall-report.md +++ b/.github/workflows/daily-firewall-report.md @@ -167,7 +167,7 @@ Use the `logs` tool from the agentic-workflows MCP server to efficiently collect **Using the logs tool:** Call the `logs` tool with the following parameters: -- `firewall`: "true" (to filter only runs with firewall enabled) +- `firewall`: true (boolean - to filter only runs with firewall enabled) - `start_date`: "-7d" (to get runs from the past 7 days) - `count`: 100 (to get up to 100 matching runs) @@ -180,7 +180,7 @@ The tool will: **Tool call example:** ```json { - "firewall": "true", + "firewall": true, "start_date": "-7d", "count": 100 } diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index c8888a815b1..f41defb0446 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -327,8 +327,8 @@ Examples: ` + constants.CLIExtensionPrefix + ` logs --engine claude # Filter logs by claude engine ` + constants.CLIExtensionPrefix + ` logs --engine codex # Filter logs by codex engine ` + constants.CLIExtensionPrefix + ` logs --engine copilot # Filter logs by copilot engine - ` + constants.CLIExtensionPrefix + ` logs --firewall true # Filter logs with firewall enabled - ` + constants.CLIExtensionPrefix + ` logs --firewall false # Filter logs without firewall + ` + constants.CLIExtensionPrefix + ` logs --firewall # Filter logs with firewall enabled + ` + constants.CLIExtensionPrefix + ` logs --no-firewall # Filter logs without firewall ` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory ` + constants.CLIExtensionPrefix + ` logs --branch main # Filter logs by branch name ` + constants.CLIExtensionPrefix + ` logs --branch feature-xyz # Filter logs by feature branch @@ -373,7 +373,8 @@ Examples: verbose, _ := cmd.Flags().GetBool("verbose") toolGraph, _ := cmd.Flags().GetBool("tool-graph") noStaged, _ := cmd.Flags().GetBool("no-staged") - firewallFilter, _ := cmd.Flags().GetString("firewall") + firewallOnly, _ := cmd.Flags().GetBool("firewall") + noFirewall, _ := cmd.Flags().GetBool("no-firewall") parse, _ := cmd.Flags().GetBool("parse") jsonOutput, _ := cmd.Flags().GetBool("json") timeout, _ := cmd.Flags().GetInt("timeout") @@ -416,16 +417,16 @@ Examples: } } - // Validate firewall parameter - if firewallFilter != "" && firewallFilter != "true" && firewallFilter != "false" { + // Validate firewall parameters + if firewallOnly && noFirewall { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", - Message: fmt.Sprintf("invalid firewall value '%s'. Must be 'true' or 'false'", firewallFilter), + Message: "cannot specify both --firewall and --no-firewall flags", })) os.Exit(1) } - if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, branch, beforeRunID, afterRunID, verbose, toolGraph, noStaged, firewallFilter, parse, jsonOutput, timeout); err != nil { + if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, branch, beforeRunID, afterRunID, verbose, toolGraph, noStaged, firewallOnly, noFirewall, parse, jsonOutput, timeout); err != nil { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", Message: err.Error(), @@ -446,7 +447,8 @@ Examples: logsCmd.Flags().Int64("after-run-id", 0, "Filter runs with database ID after this value (exclusive)") logsCmd.Flags().Bool("tool-graph", false, "Generate Mermaid tool sequence graph from agent logs") logsCmd.Flags().Bool("no-staged", false, "Filter out staged workflow runs (exclude runs with staged: true in aw_info.json)") - logsCmd.Flags().String("firewall", "", "Filter runs by firewall usage: 'true' (only runs with firewall) or 'false' (only runs without firewall)") + logsCmd.Flags().Bool("firewall", false, "Filter to only runs with firewall enabled") + logsCmd.Flags().Bool("no-firewall", false, "Filter to only runs without firewall enabled") logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing markdown to log.md and firewall.md") logsCmd.Flags().Bool("json", false, "Output logs data as JSON instead of formatted console tables") logsCmd.Flags().Int("timeout", 0, "Maximum time in seconds to spend downloading logs (0 = no timeout)") @@ -455,7 +457,7 @@ Examples: } // DownloadWorkflowLogs downloads and analyzes workflow logs with metrics -func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine, branch string, beforeRunID, afterRunID int64, verbose bool, toolGraph bool, noStaged bool, firewallFilter string, parse bool, jsonOutput bool, timeout int) error { +func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine, branch string, beforeRunID, afterRunID int64, verbose bool, toolGraph bool, noStaged bool, firewallOnly bool, noFirewall bool, parse bool, jsonOutput bool, timeout int) error { logsLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s", workflowName, count, startDate, endDate, outputDir) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Fetching workflow runs from GitHub Actions...")) @@ -573,10 +575,19 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou continue } + // Parse aw_info.json once for all filters that need it (optimization) + var awInfo *AwInfo + var awInfoErr error + awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") + + // Only parse if we need it for any filter + if engine != "" || noStaged || firewallOnly || noFirewall { + awInfo, awInfoErr = parseAwInfo(awInfoPath, verbose) + } + // Apply engine filtering if specified if engine != "" { // Check if the run's engine matches the filter - awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") detectedEngine := extractEngineFromAwInfo(awInfoPath, verbose) var engineMatches bool @@ -612,12 +623,9 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou // Apply staged filtering if --no-staged flag is specified if noStaged { - // Check if the run is staged - awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") - info, err := parseAwInfo(awInfoPath, verbose) var isStaged bool - if err == nil && info != nil { - isStaged = info.Staged + if awInfoErr == nil && awInfo != nil { + isStaged = awInfo.Staged } if isStaged { @@ -628,28 +636,24 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou } } - // Apply firewall filtering if --firewall flag is specified - if firewallFilter != "" { - // Check the firewall field in aw_info.json - awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") - info, err := parseAwInfo(awInfoPath, verbose) + // Apply firewall filtering if --firewall or --no-firewall flag is specified + if firewallOnly || noFirewall { var hasFirewall bool - if err == nil && info != nil { + if awInfoErr == nil && awInfo != nil { // Firewall is enabled if steps.firewall is non-empty (e.g., "squid") - hasFirewall = info.Steps.Firewall != "" + hasFirewall = awInfo.Steps.Firewall != "" } // Check if the run matches the filter - filterRequiresFirewall := firewallFilter == "true" - if filterRequiresFirewall && !hasFirewall { + if firewallOnly && !hasFirewall { if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall true)", result.Run.DatabaseID))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall)", result.Run.DatabaseID))) } continue } - if !filterRequiresFirewall && hasFirewall { + if noFirewall && hasFirewall { if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --firewall false)", result.Run.DatabaseID))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --no-firewall)", result.Run.DatabaseID))) } continue } diff --git a/pkg/cli/logs_test.go b/pkg/cli/logs_test.go index 9e3756cddf3..81ce1368b1e 100644 --- a/pkg/cli/logs_test.go +++ b/pkg/cli/logs_test.go @@ -19,7 +19,7 @@ func TestDownloadWorkflowLogs(t *testing.T) { // Test the DownloadWorkflowLogs function // This should either fail with auth error (if not authenticated) // or succeed with no results (if authenticated but no workflows match) - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", "", 0, 0, false, false, false, "", false, false, 0) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", "", 0, 0, false, false, false, false, false, false, false, 0) // If GitHub CLI is authenticated, the function may succeed but find no results // If not authenticated, it should return an auth error @@ -915,7 +915,7 @@ func TestDownloadWorkflowLogsWithEngineFilter(t *testing.T) { if !tt.expectError { // For valid engines, test that the function can be called without panic // It may still fail with auth errors, which is expected - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, "", 0, 0, false, false, false, "", false, false, 0) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, "", 0, 0, false, false, false, false, false, false, false, 0) // Clean up any created directories os.RemoveAll("./test-logs") diff --git a/pkg/cli/mcp_server.go b/pkg/cli/mcp_server.go index 262204142ec..d7e9d49afe3 100644 --- a/pkg/cli/mcp_server.go +++ b/pkg/cli/mcp_server.go @@ -243,7 +243,8 @@ Note: Output can be filtered using the jq parameter.`, StartDate string `json:"start_date,omitempty" jsonschema:"Filter runs created after this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)"` EndDate string `json:"end_date,omitempty" jsonschema:"Filter runs created before this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)"` Engine string `json:"engine,omitempty" jsonschema:"Filter logs by agentic engine type (claude, codex, copilot)"` - Firewall string `json:"firewall,omitempty" jsonschema:"Filter runs by firewall usage: 'true' (only runs with firewall) or 'false' (only runs without firewall)"` + Firewall bool `json:"firewall,omitempty" jsonschema:"Filter to only runs with firewall enabled"` + NoFirewall bool `json:"no_firewall,omitempty" jsonschema:"Filter to only runs without firewall enabled"` Branch string `json:"branch,omitempty" jsonschema:"Filter runs by branch name"` AfterRunID int64 `json:"after_run_id,omitempty" jsonschema:"Filter runs with database ID after this value (exclusive)"` BeforeRunID int64 `json:"before_run_id,omitempty" jsonschema:"Filter runs with database ID before this value (exclusive)"` @@ -269,6 +270,15 @@ to filter the output to a manageable size, or adjust the 'max_tokens' parameter. - .runs[:5] (get first 5 runs) - .runs | map(select(.conclusion == "failure")) (get only failed runs)`, }, func(ctx context.Context, req *mcp.CallToolRequest, args logsArgs) (*mcp.CallToolResult, any, error) { + // Validate firewall parameters + if args.Firewall && args.NoFirewall { + return &mcp.CallToolResult{ + Content: []mcp.Content{ + &mcp.TextContent{Text: "Error: cannot specify both 'firewall' and 'no_firewall' parameters"}, + }, + }, nil, nil + } + // Build command arguments // Force output directory to /tmp/gh-aw/aw-mcp/logs for MCP server cmdArgs := []string{"logs", "-o", "/tmp/gh-aw/aw-mcp/logs"} @@ -287,8 +297,11 @@ to filter the output to a manageable size, or adjust the 'max_tokens' parameter. if args.Engine != "" { cmdArgs = append(cmdArgs, "--engine", args.Engine) } - if args.Firewall != "" { - cmdArgs = append(cmdArgs, "--firewall", args.Firewall) + if args.Firewall { + cmdArgs = append(cmdArgs, "--firewall") + } + if args.NoFirewall { + cmdArgs = append(cmdArgs, "--no-firewall") } if args.Branch != "" { cmdArgs = append(cmdArgs, "--branch", args.Branch)