From 58e042c6951cba2fa54713da28a6da73a5222dcb Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 00:07:39 +0200 Subject: [PATCH 1/3] fix(cli): bump libchdb to v26.7.3 so macOS arm64 dyld loads it v26.1.0's macos-arm64 libchdb.so has its LC_SYMTAB string pool at file offset 0x13148434, which is not 8-byte aligned. Newer dyld refuses it ("mis-aligned LINKEDIT string pool"), so `maple start` fails before it opens chDB. This was ~205k maple-cli ServerError events on 2026-10-03, across 0.0.11 (Homebrew), 0.0.18 and 0.0.22. v26.7.3's string pool is aligned on both macOS arches and stays aligned after the release workflow's ad-hoc codesign. Smoke-tested from source on the new lib: fresh bootstrap, reopen of an existing store, OTLP ingest, services/errors queries, first checkpoint, clean stop. Existing local stores were written by v26.1.0, so the store-version gate asks for `maple start --reset` on first start after updating. --- .../test/native-archive-crash-recovery-probe.sh | 4 ++-- apps/cli/test/native-archive-gc-probe.sh | 2 +- apps/cli/test/native-local-store-migration.sh | 2 +- scripts/build-local-binary.sh | 15 +++++++++------ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/cli/test/native-archive-crash-recovery-probe.sh b/apps/cli/test/native-archive-crash-recovery-probe.sh index ea2b3d5edc..5f855aad74 100755 --- a/apps/cli/test/native-archive-crash-recovery-probe.sh +++ b/apps/cli/test/native-archive-crash-recovery-probe.sh @@ -25,13 +25,13 @@ PORT="${2:-45291}" REPO="$(cd "$(dirname "$0")/../../.." && pwd)" WORKER="$REPO/apps/cli/test/probes/archive-crash-worker.ts" RECONCILE="$REPO/apps/cli/test/probes/archive-reconcile-worker.ts" -# The bundled `maple` binary bakes __CHDB_VERSION__=v26.1.0 via bun --define at +# The bundled `maple` binary bakes __CHDB_VERSION__=v26.7.3 via bun --define at # compile time; running the workers from SOURCE defaults CHDB_VERSION to "dev", # which makes the restore version-check reject a checkpoint made by the bundle. # Define it to match so the source-tree workers are version-consistent with the # bundle's checkpoints. (The deployed binary is consistent by construction.) CHDB_VER="$("$MAPLE" --version 2>/dev/null | grep -oE 'chdb v[^ ]+' | sed 's/chdb //')" -[ -z "$CHDB_VER" ] && CHDB_VER="v26.1.0" +[ -z "$CHDB_VER" ] && CHDB_VER="v26.7.3" BUN=(bun --define "__CHDB_VERSION__=\"${CHDB_VER}\"") command -v duckdb >/dev/null 2>&1 || { echo "FAIL: duckdb required" >&2; exit 1; } diff --git a/apps/cli/test/native-archive-gc-probe.sh b/apps/cli/test/native-archive-gc-probe.sh index ed0f2587ab..0a5c5c889f 100755 --- a/apps/cli/test/native-archive-gc-probe.sh +++ b/apps/cli/test/native-archive-gc-probe.sh @@ -39,7 +39,7 @@ command -v duckdb >/dev/null 2>&1 || { echo "FAIL: duckdb required" >&2; exit 1; [ -f "$LIBCHDB" ] || { echo "FAIL: libchdb not found at $LIBCHDB" >&2; exit 1; } CHDB_VER="$("$MAPLE" --version 2>/dev/null | grep -oE 'chdb v[^ ]+' | sed 's/chdb //')" -[ -z "$CHDB_VER" ] && CHDB_VER="v26.1.0" +[ -z "$CHDB_VER" ] && CHDB_VER="v26.7.3" BUN=(bun --define "__CHDB_VERSION__=\"${CHDB_VER}\"") pass=0 diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 9394385793..e741e4d409 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -123,7 +123,7 @@ bounded 60 "legacy source fixture" \ # compatibility evidence the v0 resolver is allowed to use. step "installing legacy marker" CHDB_VERSION="$(bounded 60 "maple --version" env MAPLE_LIBCHDB="$LIBCHDB" "$MAPLE" --version 2>/dev/null | sed -n 's/.*chdb \([^ ]*\).*/\1/p')" -[[ -n "$CHDB_VERSION" ]] || CHDB_VERSION="v26.1.0" +[[ -n "$CHDB_VERSION" ]] || CHDB_VERSION="v26.7.3" printf '%s\n' "{\"chdb\":\"$CHDB_VERSION\",\"maple\":\"native-probe\",\"createdAt\":\"unknown\",\"schema\":\"428701854f9fd30e\"}" >"$ROOT/maple-store-version.json" chmod 600 "$ROOT/maple-store-version.json" diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index c71650d101..b1321bcd64 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -10,7 +10,7 @@ # 4. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single # executable with `bun build --compile`. The schema artifacts and SPA are # embedded; the OTLP encoders run in-process; chDB is reached via bun:ffi. -# 5. Download libchdb (v26.1.0, matching what we test against) for the host +# 5. Download libchdb (v26.7.3, matching what we test against) for the host # platform and place it beside the binary. At runtime `maple` dlopens the # sibling libchdb (resolved relative to its own path) — no rpath tricks. # 6. Restore the committed ui-embed.gen.ts stub so the tree stays clean. @@ -25,7 +25,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OUT_DIR="${OUT_DIR:-$REPO_ROOT/dist}" -LIBCHDB_VERSION="${LIBCHDB_VERSION:-v26.1.0}" +LIBCHDB_VERSION="${LIBCHDB_VERSION:-v26.7.3}" UI_EMBED="$REPO_ROOT/apps/cli/src/server/ui-embed.gen.ts" # Version baked into the binary via `bun build --define`. The release workflow @@ -66,11 +66,14 @@ esac # are mutable even for a fixed tag, so this is verified before extraction. # Bumping LIBCHDB_VERSION means re-hashing each asset here. A `case` rather than # an associative array because macOS still ships bash 3.2. +# v26.1.0's macos-arm64 dylib put its LINKEDIT string pool at a non-8-byte-aligned +# offset, which newer dyld rejects ("mis-aligned LINKEDIT string pool"). Check +# `otool -l libchdb.so` (LC_SYMTAB stroff % 8 == 0) before pinning a new release. case "$ASSET" in - linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="3f192cb85cbdb6153622db8ad6f6a25d7163bbda57f283e5ba8fd22cadd0da81" ;; - linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="08ee80488822dfa04776841bb5a4b7ddce35e161ef579b5615b80a88b177428d" ;; - macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="425db8a197288e1826b60958b3e233d0a7a5b4b46b8065c8bb7489366dbaf47f" ;; - macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="78cfd42203cd33cb0304952f4a6c39921fd9a583c614c2eef96ef6060e9677eb" ;; + linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="bc33260c32acf78eade2ac41a9115f38e00404651fa42e3bb4c419e4f011c031" ;; + linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="d153adad1ff39b2e3caf0417f09d8bd9edd41939c7c67a3c4978a61e73fb9227" ;; + macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="af5ded3ed3e84c31af1cd198dcf459f11d2b6aad4f6ddeccc04b8a519b0300fc" ;; + macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="5640e50dccf711bf3dd5551333d08e43f433edf7bd94b2289f36c2539e627762" ;; *) echo "ERROR: no pinned sha256 for $ASSET (LIBCHDB_VERSION=$LIBCHDB_VERSION)" >&2; exit 1 ;; esac From 9b4278e6fa349397d0071b46678f8de00091d08c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 00:07:44 +0200 Subject: [PATCH 2/3] fix(ai): stop agent tool failures the model recovers from opening error issues The agent engine closes every failed execute_tool span as Error, and error_events_mv turns each into an error-issue occurrence. Most were the model writing bad SQL and correcting itself (failure_handling = returned-to-model): 437 ToolCallFailed occurrences on maple-chat since 2026-10-01. The chat turn runtime now wraps the tracer so a returned tool failure exports as Ok with error.type set. error.type keeps it counted as a failed call in Agent Sessions and the tool overview (the gateway stamps maple_ai.error from it). Propagated failures keep their Error span. Also name the traces table in run_sql's description: otel_spans, otel_traces and spans were the most common guessed table names. --- apps/ai/src/chat/turn-runner.ts | 3 +- apps/ai/src/mcp/tools/run-sql.ts | 3 +- apps/ai/src/platform/genai-spans.test.ts | 51 +++++++++++++++++++++++- apps/ai/src/platform/genai-spans.ts | 38 +++++++++++++++++- 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/apps/ai/src/chat/turn-runner.ts b/apps/ai/src/chat/turn-runner.ts index 3431013f13..f7e6fe0001 100644 --- a/apps/ai/src/chat/turn-runner.ts +++ b/apps/ai/src/chat/turn-runner.ts @@ -26,6 +26,7 @@ import { workerEnvLayer } from "@maple/infra/worker-runtime" import { workerTelemetryConfig } from "@maple/infra/worker-telemetry" import { Cause, Effect, Exit, Layer, ManagedRuntime, Match, Option } from "effect" import type { WorkersAiBinding } from "../platform/WorkersAiHttpClient" +import { ReturnedToolFailuresOkLayer } from "../platform/genai-spans" import type { ChatSession } from "./ChatSession" import type { ChatTurnEvent } from "./events" import { withToolTranscript } from "./close-out" @@ -252,7 +253,7 @@ export const runChatSessionTurn = async (input: RunChatSessionTurnInput): Promis Layer.provideMerge(layerPg), Layer.provideMerge(mapleDbConnectionLayer(input.env)), Layer.provideMerge(workerEnvLayer(input.env)), - Layer.provideMerge(telemetry.layer), + Layer.provideMerge(ReturnedToolFailuresOkLayer.pipe(Layer.provideMerge(telemetry.layer))), ), ) diff --git a/apps/ai/src/mcp/tools/run-sql.ts b/apps/ai/src/mcp/tools/run-sql.ts index cdd1b242a1..d00f4050dc 100644 --- a/apps/ai/src/mcp/tools/run-sql.ts +++ b/apps/ai/src/mcp/tools/run-sql.ts @@ -109,7 +109,8 @@ const runSqlDescription = `at most ${MAX_RAW_SQL_RESULT_ROWS} are fetched and the first ${MAX_RENDERED_ROWS} rendered, with the true count reported. ` + "Use it to answer what query_data cannot express, to spot-check data, or to test SQL before saving it " + "as a raw_sql widget with add_dashboard_widget. For trends and top-N prefer query_data. " + - "describe_warehouse_tables gives table and column names." + "Spans are in `traces` (there is no otel_spans, otel_traces or spans table) and log lines in `logs`; " + + "describe_warehouse_tables gives every table's columns." /** A warehouse cell as JSON: 64-bit ints can arrive as bigint, and nothing else is JSON-unsafe. */ const toJson = (value: unknown): Schema.Json => { diff --git a/apps/ai/src/platform/genai-spans.test.ts b/apps/ai/src/platform/genai-spans.test.ts index 5aeda01422..89535092e8 100644 --- a/apps/ai/src/platform/genai-spans.test.ts +++ b/apps/ai/src/platform/genai-spans.test.ts @@ -3,10 +3,16 @@ * and tool results run to 50k, so these paths run in production far more often than the happy one. */ import { assert, describe, it } from "@effect/vitest" -import { Effect, Schema } from "effect" +import { Effect, Exit, Schema } from "effect" import { Tool } from "effect/ai" import { makeRecordingTracer } from "@maple/backend/testing/recording-tracer" -import { invokeAgentAttributes, messagesJson, toolCallJson, withToolCallContent } from "./genai-spans" +import { + invokeAgentAttributes, + messagesJson, + toolCallJson, + withReturnedToolFailuresOk, + withToolCallContent, +} from "./genai-spans" const TRUNCATION_MARKER = "…[truncated]" @@ -175,3 +181,44 @@ describe("withToolCallContent", () => { }), ) }) + +describe("withReturnedToolFailuresOk", () => { + // The engine writes `failure_handling` on the span before it ends it with a failed exit. + const endedSpan = (name: string, failureHandling: string) => + Effect.gen(function* () { + const { spans, tracer } = makeRecordingTracer() + yield* Effect.annotateCurrentSpan({ "effect_agent.tool.failure_handling": failureHandling }).pipe( + Effect.andThen(Effect.fail("bad sql")), + Effect.withSpan(name), + Effect.withTracer(withReturnedToolFailuresOk(tracer)), + Effect.exit, + ) + const span = spans[0] + assert.isDefined(span) + assert.strictEqual(span.status._tag, "Ended") + return { span, failed: span.status._tag === "Ended" && Exit.isFailure(span.status.exit) } + }) + + it.effect("ends a failure returned to the model as Ok, with error.type", () => + Effect.gen(function* () { + const { span, failed } = yield* endedSpan("execute_tool run_sql", "returned-to-model") + assert.isFalse(failed) + assert.strictEqual(span.attributes.get("error.type"), "ToolCallFailed") + }), + ) + + it.effect("keeps a propagated failure as Error", () => + Effect.gen(function* () { + const { span, failed } = yield* endedSpan("execute_tool run_sql", "propagated") + assert.isTrue(failed) + assert.isUndefined(span.attributes.get("error.type")) + }), + ) + + it.effect("leaves spans other than execute_tool alone", () => + Effect.gen(function* () { + const { failed } = yield* endedSpan("chat.turn", "returned-to-model") + assert.isTrue(failed) + }), + ) +}) diff --git a/apps/ai/src/platform/genai-spans.ts b/apps/ai/src/platform/genai-spans.ts index cd28ff6a7f..3e0aeed6a1 100644 --- a/apps/ai/src/platform/genai-spans.ts +++ b/apps/ai/src/platform/genai-spans.ts @@ -18,8 +18,7 @@ import { MAPLE_GENAI_INPUT_MESSAGES_DROPPED_ATTR, MAPLE_GENAI_MODEL_DURATION_MS_ATTR, } from "@maple/domain/gen-ai" -import { Effect, Option, Predicate, Stream } from "effect" -import type { Tracer } from "effect" +import { Effect, Exit, Layer, Option, Predicate, Stream, Tracer } from "effect" import * as AiError from "effect/ai/AiError" import type * as LanguageModel from "effect/ai/LanguageModel" import type * as Prompt from "effect/ai/Prompt" @@ -525,6 +524,41 @@ const executeToolSpan = (span: Tracer.Span): Option.Option => ? Option.some(span) : Option.filter(span.parent, isExecuteToolSpan) +/** A tool failure handed back to the model as the call's result: it retries, the run carries on. */ +const isReturnedToolFailure = (span: Tracer.Span, exit: Exit.Exit): boolean => + Exit.isFailure(exit) && + span.name.startsWith(EXECUTE_TOOL_SPAN_PREFIX) && + span.attributes.get("effect_agent.tool.failure_handling") === "returned-to-model" + +/** + * Export a returned tool failure as `Ok` + `error.type` rather than `Error`, so it stops opening error + * issues while Agent Sessions still counts it failed (the gateway stamps `maple_ai.error` from + * `error.type`). Propagated failures keep their Error span. + */ +export const withReturnedToolFailuresOk = (tracer: Tracer.Tracer): Tracer.Tracer => + Tracer.make({ + ...(tracer.context === undefined ? undefined : { context: tracer.context }), + span(options) { + const span = tracer.span(options) + if (!options.name.startsWith(EXECUTE_TOOL_SPAN_PREFIX)) return span + const end = span.end.bind(span) + return Object.assign(span, { + end: (endTime: bigint, exit: Exit.Exit) => { + if (!isReturnedToolFailure(span, exit)) return end(endTime, exit) + if (span.attributes.get("error.type") === undefined) + span.attribute("error.type", "ToolCallFailed") + end(endTime, Exit.void) + }, + }) + }, + }) + +/** Wraps the current tracer; put it over the telemetry layer with `Layer.provideMerge`. */ +export const ReturnedToolFailuresOkLayer = Layer.effect( + Tracer.Tracer, + Effect.map(Effect.tracer, withReturnedToolFailuresOk), +) + /** * Untraced on purpose: a span of its own would become the current span, one level further from the * `execute_tool` span this looks for. From 6a308f35c29031353875658813f82e3217532558 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 00:22:52 +0200 Subject: [PATCH 3/3] fix(cli): pick the libchdb checksum by version and asset LIBCHDB_VERSION can be overridden from the environment, but the sha256 was selected by platform asset only, so any other version was checked against the default release's hash and always failed. Key the case on version/asset and reject an unpinned pair before downloading. --- scripts/build-local-binary.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index b1321bcd64..1f841b43da 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -62,19 +62,19 @@ case "$(uname -s)-$(uname -m)" in Darwin-arm64) ASSET="macos-arm64-libchdb.tar.gz" ;; *) echo "ERROR: unsupported platform $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac -# Expected sha256 per platform asset, pinned to LIBCHDB_VERSION: release assets -# are mutable even for a fixed tag, so this is verified before extraction. -# Bumping LIBCHDB_VERSION means re-hashing each asset here. A `case` rather than -# an associative array because macOS still ships bash 3.2. +# Expected sha256 per (LIBCHDB_VERSION, platform asset): release assets are +# mutable even for a fixed tag, so this is verified before extraction. Bumping +# LIBCHDB_VERSION means adding each asset's hash here. A `case` rather than an +# associative array because macOS still ships bash 3.2. # v26.1.0's macos-arm64 dylib put its LINKEDIT string pool at a non-8-byte-aligned # offset, which newer dyld rejects ("mis-aligned LINKEDIT string pool"). Check # `otool -l libchdb.so` (LC_SYMTAB stroff % 8 == 0) before pinning a new release. -case "$ASSET" in - linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="bc33260c32acf78eade2ac41a9115f38e00404651fa42e3bb4c419e4f011c031" ;; - linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="d153adad1ff39b2e3caf0417f09d8bd9edd41939c7c67a3c4978a61e73fb9227" ;; - macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="af5ded3ed3e84c31af1cd198dcf459f11d2b6aad4f6ddeccc04b8a519b0300fc" ;; - macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="5640e50dccf711bf3dd5551333d08e43f433edf7bd94b2289f36c2539e627762" ;; - *) echo "ERROR: no pinned sha256 for $ASSET (LIBCHDB_VERSION=$LIBCHDB_VERSION)" >&2; exit 1 ;; +case "$LIBCHDB_VERSION/$ASSET" in + v26.7.3/linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="bc33260c32acf78eade2ac41a9115f38e00404651fa42e3bb4c419e4f011c031" ;; + v26.7.3/linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="d153adad1ff39b2e3caf0417f09d8bd9edd41939c7c67a3c4978a61e73fb9227" ;; + v26.7.3/macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="af5ded3ed3e84c31af1cd198dcf459f11d2b6aad4f6ddeccc04b8a519b0300fc" ;; + v26.7.3/macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="5640e50dccf711bf3dd5551333d08e43f433edf7bd94b2289f36c2539e627762" ;; + *) echo "ERROR: no pinned sha256 for $ASSET at LIBCHDB_VERSION=$LIBCHDB_VERSION" >&2; exit 1 ;; esac URL="https://github.com/chdb-io/chdb-core/releases/download/$LIBCHDB_VERSION/$ASSET"