ai_ops/tests/claude-observability.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, readFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
import { ClaudeObservabilityLogger, summarizeClaudeMessage } from "../src/ui/claude-observability.js";

test("summarizeClaudeMessage returns compact result metadata in summary mode", () => {
  const message = {
    type: "result",
    subtype: "success",
    stop_reason: "end_turn",
    num_turns: 1,
    total_cost_usd: 0.0012,
    usage: {
      input_tokens: 120,
      output_tokens: 40,
    },
    result: "{\"status\":\"success\"}",
    duration_ms: 40,
    duration_api_ms: 32,
    is_error: false,
    modelUsage: {},
    permission_denials: [],
    uuid: "uuid-1",
    session_id: "sdk-session-1",
  } as unknown as SDKMessage;

  const summary = summarizeClaudeMessage(message, "summary");

  assert.equal(summary.messageType, "result");
  assert.equal(summary.messageSubtype, "success");
  assert.equal(summary.sdkSessionId, "sdk-session-1");
  assert.equal(summary.summary, "Claude query result success.");
  assert.equal(summary.data?.numTurns, 1);
  const usage = summary.data?.usage as Record<string, unknown> | undefined;
  assert.equal(usage?.input_tokens, 120);
});

test("summarizeClaudeMessage redacts sensitive fields in full mode", () => {
  const message = {
    type: "system",
    subtype: "init",
    session_id: "sdk-session-2",
    uuid: "uuid-2",
    apiKey: "top-secret",
    nested: {
      authToken: "really-secret",
      ok: true,
    },
  } as unknown as SDKMessage;

  const summary = summarizeClaudeMessage(message, "full");
  const payload = summary.data?.message as Record<string, unknown> | undefined;
  const nested = payload?.nested as Record<string, unknown> | undefined;

  assert.equal(summary.messageType, "system");
  assert.equal(summary.messageSubtype, "init");
  assert.equal(payload?.apiKey, "[redacted]");
  assert.equal(nested?.authToken, "[redacted]");
  assert.equal(nested?.ok, true);
});

test("ClaudeObservabilityLogger samples tool_progress messages for stdout", () => {
  const lines: string[] = [];
  const originalLog = console.log;
  const originalNow = Date.now;
  let now = 1000;

  console.log = (line?: unknown) => {
    lines.push(String(line ?? ""));
  };
  Date.now = () => now;

  try {
    const logger = new ClaudeObservabilityLogger({
      workspaceRoot: process.cwd(),
      config: {
        mode: "stdout",
        verbosity: "summary",
        logPath: ".ai_ops/events/claude-trace.ndjson",
        includePartialMessages: false,
        debug: false,
      },
    });

    const context = {
      sessionId: "session-a",
      nodeId: "node-a",
      attempt: 1,
      depth: 0,
    };

    const makeMessage = (): SDKMessage =>
      ({
        type: "tool_progress",
        tool_name: "Bash",
        tool_use_id: "tool-1",
        parent_tool_use_id: null,
        elapsed_time_seconds: 1,
        uuid: "uuid-tool",
        session_id: "sdk-session-tool",
      }) as unknown as SDKMessage;

    logger.recordMessage({
      context,
      message: makeMessage(),
    });

    now += 300;
    logger.recordMessage({
      context,
      message: makeMessage(),
    });

    now += 1200;
    logger.recordMessage({
      context,
      message: makeMessage(),
    });

    assert.equal(lines.length, 2);
    assert.match(lines[0] ?? "", /^\[claude-trace\] /);
    assert.match(lines[1] ?? "", /"suppressedSinceLastEmit":1/);
  } finally {
    console.log = originalLog;
    Date.now = originalNow;
  }
});

test("ClaudeObservabilityLogger keeps assistant/user message records in file output", async () => {
  const workspace = await mkdtemp(join(tmpdir(), "claude-obsv-test-"));
  const logPath = ".ai_ops/events/claude-trace.ndjson";
  const logger = new ClaudeObservabilityLogger({
    workspaceRoot: workspace,
    config: {
      mode: "file",
      verbosity: "summary",
      logPath,
      includePartialMessages: false,
      debug: false,
    },
  });

  const context = {
    sessionId: "session-file",
    nodeId: "node-file",
    attempt: 1,
    depth: 0,
  };

  logger.recordQueryStarted({
    context,
  });
  logger.recordMessage({
    context,
    message: {
      type: "assistant",
      uuid: "assistant-1",
      session_id: "sdk-file-1",
      parent_tool_use_id: null,
      message: {} as never,
    } as unknown as SDKMessage,
  });
  logger.recordMessage({
    context,
    message: {
      type: "user",
      uuid: "user-1",
      session_id: "sdk-file-1",
      parent_tool_use_id: null,
      message: {} as never,
    } as unknown as SDKMessage,
  });
  logger.recordMessage({
    context,
    message: {
      type: "result",
      subtype: "success",
      stop_reason: "end_turn",
      num_turns: 1,
      total_cost_usd: 0.0012,
      usage: {
        input_tokens: 100,
        output_tokens: 20,
      },
      result: "{}",
      duration_ms: 10,
      duration_api_ms: 9,
      is_error: false,
      modelUsage: {},
      permission_denials: [],
      uuid: "result-1",
      session_id: "sdk-file-1",
    } as unknown as SDKMessage,
  });
  logger.recordQueryCompleted({
    context,
  });

  await logger.close();

  const filePath = join(workspace, logPath);
  const content = await readFile(filePath, "utf8");
  const lines = content.split(/\r?\n/).filter((line) => line.trim().length > 0);
  const records = lines.map((line) => JSON.parse(line) as Record<string, unknown>);
  const messageTypes = records
    .map((record) => record.sdkMessageType)
    .filter((value) => typeof value === "string");

  assert.equal(messageTypes.includes("assistant"), true);
  assert.equal(messageTypes.includes("user"), true);
  assert.equal(messageTypes.includes("result"), true);
});

test("summarizeClaudeMessage maps task_notification system subtype", () => {
  const message = {
    type: "system",
    subtype: "task_notification",
    task_id: "task-1",
    status: "completed",
    output_file: "/tmp/out.txt",
    summary: "Task complete",
    uuid: "uuid-task",
    session_id: "sdk-session-task",
  } as unknown as SDKMessage;

  const summary = summarizeClaudeMessage(message, "summary");

  assert.equal(summary.messageType, "system");
  assert.equal(summary.messageSubtype, "task_notification");
  assert.equal(summary.summary, "Task notification: completed.");
  assert.equal(summary.data?.taskId, "task-1");
});

test("ClaudeObservabilityLogger honors includePartialMessages for stream events", () => {
  const lines: string[] = [];
  const originalLog = console.log;
  console.log = (line?: unknown) => {
    lines.push(String(line ?? ""));
  };

  try {
    const context = {
      sessionId: "session-stream",
      nodeId: "node-stream",
      attempt: 1,
      depth: 0,
    };
    const streamMessage = {
      type: "stream_event",
      event: {
        type: "content_block_delta",
      },
      parent_tool_use_id: null,
      uuid: "stream-1",
      session_id: "sdk-session-stream",
    } as unknown as SDKMessage;

    const withoutPartial = new ClaudeObservabilityLogger({
      workspaceRoot: process.cwd(),
      config: {
        mode: "stdout",
        verbosity: "summary",
        logPath: ".ai_ops/events/claude-trace.ndjson",
        includePartialMessages: false,
        debug: false,
      },
    });
    withoutPartial.recordMessage({
      context,
      message: streamMessage,
    });

    const withPartial = new ClaudeObservabilityLogger({
      workspaceRoot: process.cwd(),
      config: {
        mode: "stdout",
        verbosity: "summary",
        logPath: ".ai_ops/events/claude-trace.ndjson",
        includePartialMessages: true,
        debug: false,
      },
    });
    withPartial.recordMessage({
      context,
      message: streamMessage,
    });

    assert.equal(lines.length, 1);
    assert.match(lines[0] ?? "", /\"sdkMessageType\":\"stream_event\"/);
  } finally {
    console.log = originalLog;
  }
});