Skip to content
This repository was archived by the owner on Sep 7, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions .github/scripts/post-endpoint-audit-report.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
const fs = require('node:fs');

const MARKER = '<!-- endpoint-audit-report -->';
const REPORT_PATH = '.audit/endpoint-audit.json';

function list(items) {
return items.length ? items.map((item) => `- \`${item}\``).join('\n') : '_none_';
}

function renderExceptionGroups(title, exceptions) {
if (!exceptions.length) return '';

const groups = new Map();
for (const exception of exceptions) {
const key = JSON.stringify({
reason: exception.reason,
owner: exception.owner,
tracking: exception.tracking,
});
const group = groups.get(key) ?? { ...exception, endpoints: [] };
group.endpoints.push(exception.endpoint);
groups.set(key, group);
}

return [
`#### ${title} (${exceptions.length})`,
'',
...[...groups.values()].flatMap((group) => [
list(group.endpoints),
'',
`Reason: ${group.reason}`,
`Owner: ${group.owner}`,
...(group.tracking ? [`Tracking: ${group.tracking}`] : []),
'',
]),
].join('\n');
}

function renderEndpointAuditReport(report) {
const clientOnly = report.clientOnly || report.mismatch || [];
const serverOnly = report.serverOnly || report.missingApi || [];
const allowedClientOnly = report.allowedClientOnly || [];
const explicitlyCovered = report.explicitlyCoveredServerOperations || [];
const actionableCount = clientOnly.length + serverOnly.length;
const documentedCount = allowedClientOnly.length + explicitlyCovered.length;
const blocking = Boolean(report.gate?.clientOnly || report.gate?.serverOnly);
const status = actionableCount
? `⚠️ ${actionableCount} actionable Agent Server contract divergence(s)`
: '✅ No actionable handwritten Agent Server contract divergences';
const mode = blocking ? 'blocking' : 'report-only';
const source = report.contractSources?.[0];
const sourceLine = source
? source.startsWith('https://')
? `Contract: [pinned release artifact](${source})`
: `Contract: \`${source}\``
: undefined;
const summary = [
'| Category | Count |',
'| --- | ---: |',
`| Actionable client-only calls | ${clientOnly.length} |`,
`| Actionable server-only operations | ${serverOnly.length} |`,
`| Documented non-divergences | ${documentedCount} |`,
`| Agent Server contract operations | ${report.agentServer} |`,
`| Audited handwritten client endpoints | ${report.client} |`,
].join('\n');
const documentedDetails = [
'<details>',
`<summary>Documented non-divergences (${documentedCount})</summary>`,
'',
renderExceptionGroups(
'Client calls intentionally absent from the filtered contract',
allowedClientOnly
),
renderExceptionGroups('Server operations covered by an exposed browser URL', explicitlyCovered),
'</details>',
].join('\n');

return [
MARKER,
'## Endpoint audit',
'',
`**${status}** · ${mode}`,
...(sourceLine ? ['', sourceLine] : []),
'',
summary,
'',
`### Actionable client-only calls (${clientOnly.length})`,
'',
list(clientOnly),
'',
`### Actionable server-only operations (${serverOnly.length})`,
'',
list(serverOnly),
...(documentedCount ? ['', documentedDetails] : []),
].join('\n');
}

async function postEndpointAuditReport({ github, context, core, reportPath = REPORT_PATH }) {
if (!fs.existsSync(reportPath)) {
core.warning(`No audit report at ${reportPath}; skipping PR comment.`);
return;
}

const body = renderEndpointAuditReport(JSON.parse(fs.readFileSync(reportPath, 'utf8')));
const { owner, repo } = context.repo;
const issue_number = context.issue.number;
const comments = await github.paginate(github.rest.issues.listComments, {
owner,
repo,
issue_number,
});
const existing = comments.find((comment) => comment.body && comment.body.includes(MARKER));

if (existing) {
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ owner, repo, issue_number, body });
}
}

module.exports = { postEndpointAuditReport, renderEndpointAuditReport };
105 changes: 10 additions & 95 deletions .github/workflows/endpoint-audit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Endpoint Audit

on:
push:
branches: [ '**' ]
branches: ['**']
# Required for the "Comment audit report on PR" step: it is gated on
# github.event_name == 'pull_request' and relies on the PR context
# (context.issue.number), neither of which exists on a push event.
Expand All @@ -14,14 +14,11 @@ permissions:
pull-requests: write

jobs:
# Verify the client doesn't call endpoints the agent-server no longer exposes.
# The audit diffs the client's HTTP surface against the live server's OpenAPI
# spec. See scripts/endpoint-audit.mjs.
# Report divergences between handwritten Agent Server calls and the pinned
# server's OpenAPI contract. See scripts/endpoint-audit.mjs.
endpoint-audit:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
AGENT_SERVER_PORT: 8010
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand All @@ -35,21 +32,10 @@ jobs:
- name: Install dependencies
run: npm ci

# Single source of truth: package.json's config.agentServerImage.
- name: Resolve agent-server image
run: echo "AGENT_SERVER_IMAGE=$(node -p "require('./package.json').config.agentServerImage")" >> "$GITHUB_ENV"
- name: Test endpoint-audit tooling
run: npm run test:endpoint-audit-tooling

- name: Start agent-server (ground-truth OpenAPI spec)
run: |
docker pull "${AGENT_SERVER_IMAGE}"
docker run -d --name agent-server -p ${AGENT_SERVER_PORT}:8000 \
-e LOG_JSON=true "${AGENT_SERVER_IMAGE}"
for i in $(seq 1 30); do
curl -sf http://localhost:${AGENT_SERVER_PORT}/health && break
echo "waiting for agent-server ($i)..."; sleep 2
done

- name: Endpoint audit (report-only — client vs agent-server drift)
- name: Endpoint audit (report-only — client vs pinned release contract)
run: npm run audit:endpoints

- name: Upload audit report
Expand All @@ -69,78 +55,7 @@ jobs:
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const reportPath = '.audit/endpoint-audit.json';
if (!fs.existsSync(reportPath)) {
core.warning(`No audit report at ${reportPath}; skipping PR comment.`);
return;
}
const r = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
const list = (items) =>
items && items.length ? items.map((i) => `- \`${i}\``).join('\n') : '_none_';
const marker = '<!-- endpoint-audit-report -->';
const NO_BACKEND = '(no known backend)';
const offContract = r.mismatch || [];
const byBackend = r.byBackend || {};
const status = offContract.length
? `❌ ${offContract.length} off-contract call(s) — not on the agent-server`
: '✅ All client calls are on the agent-server';
const backendSections = Object.keys(byBackend)
.sort()
.map((name) => {
const heading =
name === NO_BACKEND
? `#### ⛔ ${name} — served by no backend we can see (${byBackend[name].length})`
: `#### ↗️ served by \`${name}\` (${byBackend[name].length})`;
return [heading, '', list(byBackend[name]), ''].join('\n');
})
.join('\n');
const classifiers =
r.classifiers && r.classifiers.length ? ` · classifiers: ${r.classifiers.join(', ')}` : '';
const summary = [
'| Category | Count |',
'| --- | ---: |',
`| ❌ Off-contract (not on agent-server) | ${offContract.length} |`,
...Object.keys(byBackend)
.sort()
.map((name) =>
name === NO_BACKEND
? `| &nbsp;&nbsp;⛔ no known backend | ${byBackend[name].length} |`
: `| &nbsp;&nbsp;↗️ served by \`${name}\` | ${byBackend[name].length} |`
),
`| ➕ Missing API (agent-server has, client lacks) | ${(r.missingApi || []).length} |`,
`| agent-server endpoints | ${r.agentServer} |`,
`| client endpoints | ${r.client} |`,
].join('\n');
const body = [
marker,
'## Endpoint audit',
'',
`**${status}**${classifiers}`,
'',
summary,
'',
`### ❌ Not on agent-server (gated, ${offContract.length})`,
'',
offContract.length ? backendSections : '_none_',
`### ➕ Missing API — agent-server exposes it, client does not implement (${(r.missingApi || []).length})`,
'',
list(r.missingApi),
].join('\n');
const { owner, repo } = context.repo;
const issue_number = context.issue.number;
const comments = await github.paginate(github.rest.issues.listComments, {
owner,
repo,
issue_number,
});
const existing = comments.find((c) => c.body && c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ owner, repo, issue_number, body });
}

- name: Stop agent-server
if: always()
run: docker rm -f agent-server || true
const { postEndpointAuditReport } = require(
'./.github/scripts/post-endpoint-audit-report.cjs'
);
await postEndpointAuditReport({ github, context, core });
8 changes: 4 additions & 4 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ jobs:
run: |
docker run -d \
--name agent-server \
-p ${{ env.AGENT_SERVER_PORT }}:8000 \
-p 127.0.0.1:${{ env.AGENT_SERVER_PORT }}:8000 \
-v ${{ env.HOST_WORKSPACE_DIR }}:${{ env.AGENT_WORKSPACE_DIR }} \
-e LOG_JSON=true \
${{ env.AGENT_SERVER_IMAGE }}
${{ env.AGENT_SERVER_IMAGE }} --host 0.0.0.0

# Wait for container to start
sleep 5
Expand Down Expand Up @@ -182,9 +182,9 @@ jobs:
docker pull ${{ env.AGENT_SERVER_IMAGE }}
docker run -d \
--name agent-server \
-p 8010:8000 \
-p 127.0.0.1:8010:8000 \
-v /tmp/agent-workspace:/workspace \
${{ env.AGENT_SERVER_IMAGE }}
${{ env.AGENT_SERVER_IMAGE }} --host 0.0.0.0

# Wait for server
sleep 10
Expand Down
8 changes: 4 additions & 4 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -386,10 +386,10 @@ Integration tests are in `src/__tests__/integration/` and require a running agen
export LLM_API_KEY="your-api-key"
export LLM_MODEL="anthropic/claude-sonnet-4-5-20250929"

# Start agent-server in Docker (software-agent-sdk v1.37.0)
docker run -d --name agent-server -p 8010:8000 \
# Start agent-server in Docker (software-agent-sdk v1.38.0)
docker run -d --name agent-server -p 127.0.0.1:8010:8000 \
-v /tmp/agent-workspace:/workspace \
ghcr.io/openhands/agent-server:1.37.0-python
ghcr.io/openhands/agent-server:1.38.0-python --host 0.0.0.0

# Run integration tests
npm run test:integration
Expand Down Expand Up @@ -418,7 +418,7 @@ Required GitHub secrets:

### CI Image Version

- The integration workflow pins `ghcr.io/openhands/agent-server:1.37.0-python`, which corresponds to the `software-agent-sdk` release `v1.37.0`.
- The integration workflow pins `ghcr.io/openhands/agent-server:1.38.0-python`, which corresponds to the `software-agent-sdk` release `v1.38.0`.
- Keep the TypeScript client tests strict against that released server image rather than adding compatibility fallbacks for older prerelease builds.

## Agent Behavior Guidelines
Expand Down
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ issue. Ordinary client PRs never generate from the moving SDK branch.
You'll need an AgentServer running somewhere for the client to connect to. You can run one in docker:

```bash
docker run -p 8000:8000 -p 8001:8001 \
docker run -p 127.0.0.1:8000:8000 -p 127.0.0.1:8001:8001 \
-e OH_ENABLE_VNC=false \
-e SESSION_API_KEY="$SESSION_API_KEY" \
-e OH_ALLOW_CORS_ORIGINS='["*"]' \
Expand Down Expand Up @@ -448,14 +448,14 @@ Integration tests require a running agent-server in Docker with a mounted worksp
chmod 777 /tmp/agent-workspace
```

2. Start the agent-server container (software-agent-sdk v1.37.0):
2. Start the agent-server container (software-agent-sdk v1.38.0):

```bash
docker run -d \
--name agent-server \
-p 8010:8000 \
-p 127.0.0.1:8010:8000 \
-v /tmp/agent-workspace:/workspace \
ghcr.io/openhands/agent-server:1.37.0-python
ghcr.io/openhands/agent-server:1.38.0-python --host 0.0.0.0
```

3. Wait for the server to be ready:
Expand Down
41 changes: 34 additions & 7 deletions endpoint-audit.config.json
Original file line number Diff line number Diff line change
@@ -1,21 +1,48 @@
{
"_comment": "Endpoint audit config. REPORT-ONLY: the audit runs on every PR, writes .audit/endpoint-audit.json, and posts a drift report as a PR comment, but does NOT fail CI (gate.mismatch=false). The client is reconciled against the agent-server only: any client call the agent-server does not expose is reported as off-contract. Additional `classify` specs are fetched live purely to label each off-contract call with the backend that actually serves it (e.g. cloud) vs no known backend at all; if a classify spec is unreachable its labels degrade to 'no known backend'. To make this a blocking gate, set gate.mismatch=true. See scripts/endpoint-audit.mjs.",
"_comment": "Endpoint audit config. REPORT-ONLY: the audit runs on every PR, writes .audit/endpoint-audit.json, and posts actionable Agent Server contract divergences as a PR comment, but does NOT fail CI. It uses the same exact pinned release artifact as generated types. Intentional client-ahead calls and routes exposed as authenticated URLs are documented below rather than reported as divergences. Clients for other backends must be excluded explicitly. To make either direction blocking, enable the corresponding gate below. See scripts/endpoint-audit.mjs.",
"specs": [
{
"name": "agent-server",
"role": "gate",
"url": "http://localhost:8010/openapi.json"
},
{
"name": "cloud",
"role": "classify",
"url": "https://app.all-hands.dev/openapi.json"
"source": "pinned-agent-server-release"
}
],
"clientGlobs": ["src/client", "src/conversation", "src/workspace"],
"excludeClientFiles": ["src/client/cloud-client.ts", "src/client/shared-client.ts"],

"ignoreServerOnly": ["/v1/"],

"allowClientOnly": [
{
"endpoints": ["GET /", "GET /alive", "GET /health", "GET /ready", "GET /server_info"],
"reason": "Operational Agent Server endpoints intentionally excluded from the filtered public release artifact.",
"owner": "OpenHands runtime maintainers"
},
{
"endpoints": [
"DELETE /api/meta-profiles/{}",
"GET /api/meta-profiles",
"GET /api/meta-profiles/{}",
"POST /api/meta-profiles/{}",
"POST /api/meta-profiles/{}/activate"
],
"reason": "Client-ahead API stacked on the pending Agent Server meta-profiles implementation.",
"owner": "OpenHands SDK maintainers",
"tracking": "https://github.com/OpenHands/software-agent-sdk/pull/3744"
}
],

"coveredServerOperations": [
{
"endpoints": [
"GET /api/conversations/{}/workspace",
"GET /api/conversations/{}/workspace/{}"
],
"reason": "RemoteWorkspace.startWorkspaceSession exposes these authenticated URLs for browser iframe and file requests; they are not HttpClient method calls.",
"owner": "OpenHands TypeScript client maintainers"
}
],

"gate": {
"mismatch": false,
"missingApi": false
Expand Down
Loading
Loading