Add Claude observability tracing and diagnostics UI
This commit is contained in:
296
tests/claude-observability.test.ts
Normal file
296
tests/claude-observability.test.ts
Normal file
@@ -0,0 +1,296 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
|
||||
import { ClaudeObservabilityLogger, summarizeClaudeMessage } from "../src/ui/claude-observability.js";
|
||||
|
||||
test("summarizeClaudeMessage returns compact result metadata in summary mode", () => {
|
||||
const message = {
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
stop_reason: "end_turn",
|
||||
num_turns: 1,
|
||||
total_cost_usd: 0.0012,
|
||||
usage: {
|
||||
input_tokens: 120,
|
||||
output_tokens: 40,
|
||||
},
|
||||
result: "{\"status\":\"success\"}",
|
||||
duration_ms: 40,
|
||||
duration_api_ms: 32,
|
||||
is_error: false,
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
uuid: "uuid-1",
|
||||
session_id: "sdk-session-1",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "summary");
|
||||
|
||||
assert.equal(summary.messageType, "result");
|
||||
assert.equal(summary.messageSubtype, "success");
|
||||
assert.equal(summary.sdkSessionId, "sdk-session-1");
|
||||
assert.equal(summary.summary, "Claude query result success.");
|
||||
assert.equal(summary.data?.numTurns, 1);
|
||||
const usage = summary.data?.usage as Record<string, unknown> | undefined;
|
||||
assert.equal(usage?.input_tokens, 120);
|
||||
});
|
||||
|
||||
test("summarizeClaudeMessage redacts sensitive fields in full mode", () => {
|
||||
const message = {
|
||||
type: "system",
|
||||
subtype: "init",
|
||||
session_id: "sdk-session-2",
|
||||
uuid: "uuid-2",
|
||||
apiKey: "top-secret",
|
||||
nested: {
|
||||
authToken: "really-secret",
|
||||
ok: true,
|
||||
},
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "full");
|
||||
const payload = summary.data?.message as Record<string, unknown> | undefined;
|
||||
const nested = payload?.nested as Record<string, unknown> | undefined;
|
||||
|
||||
assert.equal(summary.messageType, "system");
|
||||
assert.equal(summary.messageSubtype, "init");
|
||||
assert.equal(payload?.apiKey, "[redacted]");
|
||||
assert.equal(nested?.authToken, "[redacted]");
|
||||
assert.equal(nested?.ok, true);
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger samples tool_progress messages for stdout", () => {
|
||||
const lines: string[] = [];
|
||||
const originalLog = console.log;
|
||||
const originalNow = Date.now;
|
||||
let now = 1000;
|
||||
|
||||
console.log = (line?: unknown) => {
|
||||
lines.push(String(line ?? ""));
|
||||
};
|
||||
Date.now = () => now;
|
||||
|
||||
try {
|
||||
const logger = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
|
||||
const context = {
|
||||
sessionId: "session-a",
|
||||
nodeId: "node-a",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
|
||||
const makeMessage = (): SDKMessage =>
|
||||
({
|
||||
type: "tool_progress",
|
||||
tool_name: "Bash",
|
||||
tool_use_id: "tool-1",
|
||||
parent_tool_use_id: null,
|
||||
elapsed_time_seconds: 1,
|
||||
uuid: "uuid-tool",
|
||||
session_id: "sdk-session-tool",
|
||||
}) as unknown as SDKMessage;
|
||||
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
now += 300;
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
now += 1200;
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
assert.equal(lines.length, 2);
|
||||
assert.match(lines[0] ?? "", /^\[claude-trace\] /);
|
||||
assert.match(lines[1] ?? "", /"suppressedSinceLastEmit":1/);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
Date.now = originalNow;
|
||||
}
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger keeps assistant/user message records in file output", async () => {
|
||||
const workspace = await mkdtemp(join(tmpdir(), "claude-obsv-test-"));
|
||||
const logPath = ".ai_ops/events/claude-trace.ndjson";
|
||||
const logger = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: workspace,
|
||||
config: {
|
||||
mode: "file",
|
||||
verbosity: "summary",
|
||||
logPath,
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
|
||||
const context = {
|
||||
sessionId: "session-file",
|
||||
nodeId: "node-file",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
|
||||
logger.recordQueryStarted({
|
||||
context,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "assistant",
|
||||
uuid: "assistant-1",
|
||||
session_id: "sdk-file-1",
|
||||
parent_tool_use_id: null,
|
||||
message: {} as never,
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "user",
|
||||
uuid: "user-1",
|
||||
session_id: "sdk-file-1",
|
||||
parent_tool_use_id: null,
|
||||
message: {} as never,
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
stop_reason: "end_turn",
|
||||
num_turns: 1,
|
||||
total_cost_usd: 0.0012,
|
||||
usage: {
|
||||
input_tokens: 100,
|
||||
output_tokens: 20,
|
||||
},
|
||||
result: "{}",
|
||||
duration_ms: 10,
|
||||
duration_api_ms: 9,
|
||||
is_error: false,
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
uuid: "result-1",
|
||||
session_id: "sdk-file-1",
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordQueryCompleted({
|
||||
context,
|
||||
});
|
||||
|
||||
await logger.close();
|
||||
|
||||
const filePath = join(workspace, logPath);
|
||||
const content = await readFile(filePath, "utf8");
|
||||
const lines = content.split(/\r?\n/).filter((line) => line.trim().length > 0);
|
||||
const records = lines.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
const messageTypes = records
|
||||
.map((record) => record.sdkMessageType)
|
||||
.filter((value) => typeof value === "string");
|
||||
|
||||
assert.equal(messageTypes.includes("assistant"), true);
|
||||
assert.equal(messageTypes.includes("user"), true);
|
||||
assert.equal(messageTypes.includes("result"), true);
|
||||
});
|
||||
|
||||
test("summarizeClaudeMessage maps task_notification system subtype", () => {
|
||||
const message = {
|
||||
type: "system",
|
||||
subtype: "task_notification",
|
||||
task_id: "task-1",
|
||||
status: "completed",
|
||||
output_file: "/tmp/out.txt",
|
||||
summary: "Task complete",
|
||||
uuid: "uuid-task",
|
||||
session_id: "sdk-session-task",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "summary");
|
||||
|
||||
assert.equal(summary.messageType, "system");
|
||||
assert.equal(summary.messageSubtype, "task_notification");
|
||||
assert.equal(summary.summary, "Task notification: completed.");
|
||||
assert.equal(summary.data?.taskId, "task-1");
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger honors includePartialMessages for stream events", () => {
|
||||
const lines: string[] = [];
|
||||
const originalLog = console.log;
|
||||
console.log = (line?: unknown) => {
|
||||
lines.push(String(line ?? ""));
|
||||
};
|
||||
|
||||
try {
|
||||
const context = {
|
||||
sessionId: "session-stream",
|
||||
nodeId: "node-stream",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
const streamMessage = {
|
||||
type: "stream_event",
|
||||
event: {
|
||||
type: "content_block_delta",
|
||||
},
|
||||
parent_tool_use_id: null,
|
||||
uuid: "stream-1",
|
||||
session_id: "sdk-session-stream",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const withoutPartial = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
withoutPartial.recordMessage({
|
||||
context,
|
||||
message: streamMessage,
|
||||
});
|
||||
|
||||
const withPartial = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: true,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
withPartial.recordMessage({
|
||||
context,
|
||||
message: streamMessage,
|
||||
});
|
||||
|
||||
assert.equal(lines.length, 1);
|
||||
assert.match(lines[0] ?? "", /\"sdkMessageType\":\"stream_event\"/);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
}
|
||||
});
|
||||
42
tests/claude-trace-store.test.ts
Normal file
42
tests/claude-trace-store.test.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { filterClaudeTraceEvents, readClaudeTraceEvents } from "../src/ui/claude-trace-store.js";
|
||||
|
||||
test("readClaudeTraceEvents parses and sorts ndjson records", async () => {
|
||||
const workspace = await mkdtemp(join(tmpdir(), "claude-trace-store-"));
|
||||
const logPath = join(workspace, "claude-trace.ndjson");
|
||||
await writeFile(
|
||||
logPath,
|
||||
[
|
||||
'{"timestamp":"2026-02-24T17:27:05.000Z","message":"later","sessionId":"s1"}',
|
||||
'not-json',
|
||||
'{"timestamp":"2026-02-24T17:26:00.000Z","message":"earlier","sessionId":"s1"}',
|
||||
'{"message":"missing timestamp"}',
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const events = await readClaudeTraceEvents(logPath);
|
||||
assert.equal(events.length, 2);
|
||||
assert.equal(events[0]?.message, "earlier");
|
||||
assert.equal(events[1]?.message, "later");
|
||||
});
|
||||
|
||||
test("filterClaudeTraceEvents filters by session and limit", () => {
|
||||
const events = [
|
||||
{ timestamp: "2026-02-24T17:00:00.000Z", message: "a", sessionId: "s1" },
|
||||
{ timestamp: "2026-02-24T17:01:00.000Z", message: "b", sessionId: "s2" },
|
||||
{ timestamp: "2026-02-24T17:02:00.000Z", message: "c", sessionId: "s1" },
|
||||
];
|
||||
|
||||
const filtered = filterClaudeTraceEvents(events, {
|
||||
sessionId: "s1",
|
||||
limit: 1,
|
||||
});
|
||||
|
||||
assert.equal(filtered.length, 1);
|
||||
assert.equal(filtered[0]?.message, "c");
|
||||
});
|
||||
@@ -25,6 +25,11 @@ test("loads defaults and freezes config", () => {
|
||||
"session.failed",
|
||||
]);
|
||||
assert.equal(config.provider.openAiAuthMode, "auto");
|
||||
assert.equal(config.provider.claudeObservability.mode, "off");
|
||||
assert.equal(config.provider.claudeObservability.verbosity, "summary");
|
||||
assert.equal(config.provider.claudeObservability.logPath, ".ai_ops/events/claude-trace.ndjson");
|
||||
assert.equal(config.provider.claudeObservability.includePartialMessages, false);
|
||||
assert.equal(config.provider.claudeObservability.debug, false);
|
||||
assert.equal(Object.isFrozen(config), true);
|
||||
assert.equal(Object.isFrozen(config.orchestration), true);
|
||||
});
|
||||
@@ -57,6 +62,38 @@ test("validates runtime discord severity mode", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("validates claude observability mode", () => {
|
||||
assert.throws(
|
||||
() => loadConfig({ CLAUDE_OBSERVABILITY_MODE: "stream" }),
|
||||
/CLAUDE_OBSERVABILITY_MODE must be one of/,
|
||||
);
|
||||
});
|
||||
|
||||
test("validates claude observability verbosity", () => {
|
||||
assert.throws(
|
||||
() => loadConfig({ CLAUDE_OBSERVABILITY_VERBOSITY: "verbose" }),
|
||||
/CLAUDE_OBSERVABILITY_VERBOSITY must be one of/,
|
||||
);
|
||||
});
|
||||
|
||||
test("loads claude observability settings", () => {
|
||||
const config = loadConfig({
|
||||
CLAUDE_OBSERVABILITY_MODE: "both",
|
||||
CLAUDE_OBSERVABILITY_VERBOSITY: "full",
|
||||
CLAUDE_OBSERVABILITY_LOG_PATH: ".ai_ops/debug/claude.ndjson",
|
||||
CLAUDE_OBSERVABILITY_INCLUDE_PARTIAL: "true",
|
||||
CLAUDE_OBSERVABILITY_DEBUG: "true",
|
||||
CLAUDE_OBSERVABILITY_DEBUG_LOG_PATH: ".ai_ops/debug/claude-sdk.log",
|
||||
});
|
||||
|
||||
assert.equal(config.provider.claudeObservability.mode, "both");
|
||||
assert.equal(config.provider.claudeObservability.verbosity, "full");
|
||||
assert.equal(config.provider.claudeObservability.logPath, ".ai_ops/debug/claude.ndjson");
|
||||
assert.equal(config.provider.claudeObservability.includePartialMessages, true);
|
||||
assert.equal(config.provider.claudeObservability.debug, true);
|
||||
assert.equal(config.provider.claudeObservability.debugLogPath, ".ai_ops/debug/claude-sdk.log");
|
||||
});
|
||||
|
||||
test("prefers CLAUDE_CODE_OAUTH_TOKEN over ANTHROPIC_API_KEY", () => {
|
||||
const config = loadConfig({
|
||||
CLAUDE_CODE_OAUTH_TOKEN: "oauth-token",
|
||||
|
||||
@@ -155,3 +155,41 @@ test("secure executor runs with explicit env policy", async () => {
|
||||
assert.equal(result.stdout, "ok|\n");
|
||||
assert.equal(streamedStdout, result.stdout);
|
||||
});
|
||||
|
||||
test("rules engine carries session context in tool audit events", () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const rules = new SecurityRulesEngine(
|
||||
{
|
||||
allowedBinaries: ["git"],
|
||||
worktreeRoot: "/tmp",
|
||||
protectedPaths: [],
|
||||
requireCwdWithinWorktree: true,
|
||||
rejectRelativePathTraversal: true,
|
||||
enforcePathBoundaryOnArguments: true,
|
||||
allowedEnvAssignments: [],
|
||||
blockedEnvAssignments: [],
|
||||
},
|
||||
(event) => {
|
||||
events.push(event as unknown as Record<string, unknown>);
|
||||
},
|
||||
);
|
||||
|
||||
rules.assertToolInvocationAllowed({
|
||||
tool: "git",
|
||||
toolClearance: {
|
||||
allowlist: ["git"],
|
||||
banlist: [],
|
||||
},
|
||||
context: {
|
||||
sessionId: "session-ctx",
|
||||
nodeId: "node-ctx",
|
||||
attempt: 2,
|
||||
},
|
||||
});
|
||||
|
||||
const allowedEvent = events.find((event) => event.type === "tool.invocation_allowed");
|
||||
assert.ok(allowedEvent);
|
||||
assert.equal(allowedEvent.sessionId, "session-ctx");
|
||||
assert.equal(allowedEvent.nodeId, "node-ctx");
|
||||
assert.equal(allowedEvent.attempt, 2);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, mkdir, readFile, writeFile, stat } from "node:fs/promises";
|
||||
import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
@@ -177,3 +177,54 @@ test("session worktree manager returns conflict outcome instead of throwing", as
|
||||
assert.equal(mergeOutcome.worktreePath, taskWorktreePath);
|
||||
assert.ok(mergeOutcome.conflictFiles.includes("README.md"));
|
||||
});
|
||||
|
||||
test("session worktree manager recreates a task worktree after stale metadata prune", async () => {
|
||||
const root = await mkdtemp(resolve(tmpdir(), "ai-ops-session-worktree-prune-"));
|
||||
const projectPath = resolve(root, "project");
|
||||
const worktreeRoot = resolve(root, "worktrees");
|
||||
|
||||
await mkdir(projectPath, { recursive: true });
|
||||
await git(["init", projectPath]);
|
||||
await git(["-C", projectPath, "config", "user.name", "AI Ops"]);
|
||||
await git(["-C", projectPath, "config", "user.email", "ai-ops@example.local"]);
|
||||
await writeFile(resolve(projectPath, "README.md"), "# project\n", "utf8");
|
||||
await git(["-C", projectPath, "add", "README.md"]);
|
||||
await git(["-C", projectPath, "commit", "-m", "initial commit"]);
|
||||
|
||||
const manager = new SessionWorktreeManager({
|
||||
worktreeRoot,
|
||||
baseRef: "HEAD",
|
||||
});
|
||||
|
||||
const sessionId = "session-prune-1";
|
||||
const taskId = "task-prune-1";
|
||||
const baseWorkspacePath = manager.resolveBaseWorkspacePath(sessionId);
|
||||
|
||||
await manager.initializeSessionBaseWorkspace({
|
||||
sessionId,
|
||||
projectPath,
|
||||
baseWorkspacePath,
|
||||
});
|
||||
|
||||
const initialTaskWorktreePath = (
|
||||
await manager.ensureTaskWorktree({
|
||||
sessionId,
|
||||
taskId,
|
||||
baseWorkspacePath,
|
||||
})
|
||||
).taskWorktreePath;
|
||||
|
||||
await rm(initialTaskWorktreePath, { recursive: true, force: true });
|
||||
|
||||
const recreatedTaskWorktreePath = (
|
||||
await manager.ensureTaskWorktree({
|
||||
sessionId,
|
||||
taskId,
|
||||
baseWorkspacePath,
|
||||
})
|
||||
).taskWorktreePath;
|
||||
|
||||
assert.equal(recreatedTaskWorktreePath, initialTaskWorktreePath);
|
||||
const stats = await stat(recreatedTaskWorktreePath);
|
||||
assert.equal(stats.isDirectory(), true);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user