Add Claude observability tracing and diagnostics UI
This commit is contained in:
296
tests/claude-observability.test.ts
Normal file
296
tests/claude-observability.test.ts
Normal file
@@ -0,0 +1,296 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
|
||||
import { ClaudeObservabilityLogger, summarizeClaudeMessage } from "../src/ui/claude-observability.js";
|
||||
|
||||
test("summarizeClaudeMessage returns compact result metadata in summary mode", () => {
|
||||
const message = {
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
stop_reason: "end_turn",
|
||||
num_turns: 1,
|
||||
total_cost_usd: 0.0012,
|
||||
usage: {
|
||||
input_tokens: 120,
|
||||
output_tokens: 40,
|
||||
},
|
||||
result: "{\"status\":\"success\"}",
|
||||
duration_ms: 40,
|
||||
duration_api_ms: 32,
|
||||
is_error: false,
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
uuid: "uuid-1",
|
||||
session_id: "sdk-session-1",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "summary");
|
||||
|
||||
assert.equal(summary.messageType, "result");
|
||||
assert.equal(summary.messageSubtype, "success");
|
||||
assert.equal(summary.sdkSessionId, "sdk-session-1");
|
||||
assert.equal(summary.summary, "Claude query result success.");
|
||||
assert.equal(summary.data?.numTurns, 1);
|
||||
const usage = summary.data?.usage as Record<string, unknown> | undefined;
|
||||
assert.equal(usage?.input_tokens, 120);
|
||||
});
|
||||
|
||||
test("summarizeClaudeMessage redacts sensitive fields in full mode", () => {
|
||||
const message = {
|
||||
type: "system",
|
||||
subtype: "init",
|
||||
session_id: "sdk-session-2",
|
||||
uuid: "uuid-2",
|
||||
apiKey: "top-secret",
|
||||
nested: {
|
||||
authToken: "really-secret",
|
||||
ok: true,
|
||||
},
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "full");
|
||||
const payload = summary.data?.message as Record<string, unknown> | undefined;
|
||||
const nested = payload?.nested as Record<string, unknown> | undefined;
|
||||
|
||||
assert.equal(summary.messageType, "system");
|
||||
assert.equal(summary.messageSubtype, "init");
|
||||
assert.equal(payload?.apiKey, "[redacted]");
|
||||
assert.equal(nested?.authToken, "[redacted]");
|
||||
assert.equal(nested?.ok, true);
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger samples tool_progress messages for stdout", () => {
|
||||
const lines: string[] = [];
|
||||
const originalLog = console.log;
|
||||
const originalNow = Date.now;
|
||||
let now = 1000;
|
||||
|
||||
console.log = (line?: unknown) => {
|
||||
lines.push(String(line ?? ""));
|
||||
};
|
||||
Date.now = () => now;
|
||||
|
||||
try {
|
||||
const logger = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
|
||||
const context = {
|
||||
sessionId: "session-a",
|
||||
nodeId: "node-a",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
|
||||
const makeMessage = (): SDKMessage =>
|
||||
({
|
||||
type: "tool_progress",
|
||||
tool_name: "Bash",
|
||||
tool_use_id: "tool-1",
|
||||
parent_tool_use_id: null,
|
||||
elapsed_time_seconds: 1,
|
||||
uuid: "uuid-tool",
|
||||
session_id: "sdk-session-tool",
|
||||
}) as unknown as SDKMessage;
|
||||
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
now += 300;
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
now += 1200;
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: makeMessage(),
|
||||
});
|
||||
|
||||
assert.equal(lines.length, 2);
|
||||
assert.match(lines[0] ?? "", /^\[claude-trace\] /);
|
||||
assert.match(lines[1] ?? "", /"suppressedSinceLastEmit":1/);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
Date.now = originalNow;
|
||||
}
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger keeps assistant/user message records in file output", async () => {
|
||||
const workspace = await mkdtemp(join(tmpdir(), "claude-obsv-test-"));
|
||||
const logPath = ".ai_ops/events/claude-trace.ndjson";
|
||||
const logger = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: workspace,
|
||||
config: {
|
||||
mode: "file",
|
||||
verbosity: "summary",
|
||||
logPath,
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
|
||||
const context = {
|
||||
sessionId: "session-file",
|
||||
nodeId: "node-file",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
|
||||
logger.recordQueryStarted({
|
||||
context,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "assistant",
|
||||
uuid: "assistant-1",
|
||||
session_id: "sdk-file-1",
|
||||
parent_tool_use_id: null,
|
||||
message: {} as never,
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "user",
|
||||
uuid: "user-1",
|
||||
session_id: "sdk-file-1",
|
||||
parent_tool_use_id: null,
|
||||
message: {} as never,
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordMessage({
|
||||
context,
|
||||
message: {
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
stop_reason: "end_turn",
|
||||
num_turns: 1,
|
||||
total_cost_usd: 0.0012,
|
||||
usage: {
|
||||
input_tokens: 100,
|
||||
output_tokens: 20,
|
||||
},
|
||||
result: "{}",
|
||||
duration_ms: 10,
|
||||
duration_api_ms: 9,
|
||||
is_error: false,
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
uuid: "result-1",
|
||||
session_id: "sdk-file-1",
|
||||
} as unknown as SDKMessage,
|
||||
});
|
||||
logger.recordQueryCompleted({
|
||||
context,
|
||||
});
|
||||
|
||||
await logger.close();
|
||||
|
||||
const filePath = join(workspace, logPath);
|
||||
const content = await readFile(filePath, "utf8");
|
||||
const lines = content.split(/\r?\n/).filter((line) => line.trim().length > 0);
|
||||
const records = lines.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
const messageTypes = records
|
||||
.map((record) => record.sdkMessageType)
|
||||
.filter((value) => typeof value === "string");
|
||||
|
||||
assert.equal(messageTypes.includes("assistant"), true);
|
||||
assert.equal(messageTypes.includes("user"), true);
|
||||
assert.equal(messageTypes.includes("result"), true);
|
||||
});
|
||||
|
||||
test("summarizeClaudeMessage maps task_notification system subtype", () => {
|
||||
const message = {
|
||||
type: "system",
|
||||
subtype: "task_notification",
|
||||
task_id: "task-1",
|
||||
status: "completed",
|
||||
output_file: "/tmp/out.txt",
|
||||
summary: "Task complete",
|
||||
uuid: "uuid-task",
|
||||
session_id: "sdk-session-task",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const summary = summarizeClaudeMessage(message, "summary");
|
||||
|
||||
assert.equal(summary.messageType, "system");
|
||||
assert.equal(summary.messageSubtype, "task_notification");
|
||||
assert.equal(summary.summary, "Task notification: completed.");
|
||||
assert.equal(summary.data?.taskId, "task-1");
|
||||
});
|
||||
|
||||
test("ClaudeObservabilityLogger honors includePartialMessages for stream events", () => {
|
||||
const lines: string[] = [];
|
||||
const originalLog = console.log;
|
||||
console.log = (line?: unknown) => {
|
||||
lines.push(String(line ?? ""));
|
||||
};
|
||||
|
||||
try {
|
||||
const context = {
|
||||
sessionId: "session-stream",
|
||||
nodeId: "node-stream",
|
||||
attempt: 1,
|
||||
depth: 0,
|
||||
};
|
||||
const streamMessage = {
|
||||
type: "stream_event",
|
||||
event: {
|
||||
type: "content_block_delta",
|
||||
},
|
||||
parent_tool_use_id: null,
|
||||
uuid: "stream-1",
|
||||
session_id: "sdk-session-stream",
|
||||
} as unknown as SDKMessage;
|
||||
|
||||
const withoutPartial = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: false,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
withoutPartial.recordMessage({
|
||||
context,
|
||||
message: streamMessage,
|
||||
});
|
||||
|
||||
const withPartial = new ClaudeObservabilityLogger({
|
||||
workspaceRoot: process.cwd(),
|
||||
config: {
|
||||
mode: "stdout",
|
||||
verbosity: "summary",
|
||||
logPath: ".ai_ops/events/claude-trace.ndjson",
|
||||
includePartialMessages: true,
|
||||
debug: false,
|
||||
},
|
||||
});
|
||||
withPartial.recordMessage({
|
||||
context,
|
||||
message: streamMessage,
|
||||
});
|
||||
|
||||
assert.equal(lines.length, 1);
|
||||
assert.match(lines[0] ?? "", /\"sdkMessageType\":\"stream_event\"/);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user