ai_ops/tests/session-insights.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import { mkdir, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { mkdtemp } from "node:fs/promises";
import { buildSessionGraphInsight, buildSessionSummaries } from "../src/telemetry/session-insights.js";
import { parseAgentManifest } from "../src/agents/manifest.js";

function createManifest() {
  return parseAgentManifest({
    schemaVersion: "1",
    topologies: ["sequential", "retry-unrolled"],
    personas: [
      {
        id: "planner",
        displayName: "Planner",
        systemPromptTemplate: "Plan",
        toolClearance: {
          allowlist: ["read_file"],
          banlist: [],
        },
      },
    ],
    relationships: [],
    topologyConstraints: {
      maxDepth: 3,
      maxRetries: 2,
    },
    pipeline: {
      entryNodeId: "n1",
      nodes: [
        {
          id: "n1",
          actorId: "a1",
          personaId: "planner",
          topology: { kind: "sequential" },
        },
        {
          id: "n2",
          actorId: "a2",
          personaId: "planner",
          topology: { kind: "retry-unrolled" },
        },
      ],
      edges: [
        {
          from: "n1",
          to: "n2",
          event: "validation_failed",
        },
      ],
    },
  });
}

test("buildSessionGraphInsight maps attempts, edge visits, and sandbox payload", async () => {
  const root = await mkdtemp(resolve(tmpdir(), "ai-ops-session-insights-"));
  const stateRoot = resolve(root, "state");
  const sessionId = "session-1";
  const handoffDir = resolve(stateRoot, sessionId, "handoffs");
  const runtimeLogPath = resolve(root, "runtime-events.ndjson");

  await mkdir(handoffDir, { recursive: true });
  await writeFile(
    resolve(handoffDir, "n2.json"),
    `${JSON.stringify({
      nodeId: "n2",
      fromNodeId: "n1",
      payload: {},
      createdAt: new Date().toISOString(),
    })}\n`,
    "utf8",
  );

  const lines = [
    {
      id: "1",
      timestamp: "2026-01-01T00:00:00.000Z",
      type: "session.started",
      severity: "info",
      message: "started",
      sessionId,
    },
    {
      id: "2",
      timestamp: "2026-01-01T00:00:01.000Z",
      type: "node.attempt.completed",
      severity: "info",
      message: "n1 success",
      sessionId,
      nodeId: "n1",
      attempt: 1,
      usage: { durationMs: 100, costUsd: 0.001 },
      metadata: {
        status: "success",
        executionContext: { phase: "n1", allowedTools: ["read_file"] },
      },
    },
    {
      id: "3",
      timestamp: "2026-01-01T00:00:02.000Z",
      type: "node.attempt.completed",
      severity: "warning",
      message: "n2 validation",
      sessionId,
      nodeId: "n2",
      attempt: 1,
      usage: { durationMs: 140, costUsd: 0.002 },
      metadata: {
        status: "validation_fail",
        retrySpawned: true,
        subtasks: ["fix tests"],
        executionContext: { phase: "n2", allowedTools: ["read_file"] },
      },
    },
    {
      id: "4",
      timestamp: "2026-01-01T00:00:03.000Z",
      type: "node.attempt.completed",
      severity: "info",
      message: "n2 success",
      sessionId,
      nodeId: "n2",
      attempt: 2,
      usage: { durationMs: 120, costUsd: 0.0025 },
      metadata: {
        status: "success",
        executionContext: { phase: "n2", allowedTools: ["read_file"] },
      },
    },
    {
      id: "5",
      timestamp: "2026-01-01T00:00:04.000Z",
      type: "session.completed",
      severity: "info",
      message: "completed",
      sessionId,
      metadata: {
        status: "success",
      },
    },
  ];

  await writeFile(runtimeLogPath, `${lines.map((line) => JSON.stringify(line)).join("\n")}\n`, "utf8");

  const manifest = createManifest();
  const graph = await buildSessionGraphInsight({
    stateRoot,
    runtimeEventLogPath: runtimeLogPath,
    sessionId,
    manifest,
  });

  assert.equal(graph.status, "success");
  assert.equal(graph.nodes.length, 2);

  const node2 = graph.nodes.find((node: any) => node.nodeId === "n2");
  assert.ok(node2);
  assert.equal(node2.attemptCount, 2);
  assert.equal(node2.subtaskCount, 1);
  assert.equal(node2.sandboxPayload?.phase, "n2");

  const edge = graph.edges.find((entry: any) => entry.from === "n1" && entry.to === "n2");
  assert.ok(edge);
  assert.equal(edge.visited, true);
  assert.equal(edge.trigger, "event:validation_failed");
});

test("buildSessionSummaries reflects aborted failed session", async () => {
  const root = await mkdtemp(resolve(tmpdir(), "ai-ops-session-insights-"));
  const stateRoot = resolve(root, "state");
  const sessionId = "session-abort";
  const runtimeLogPath = resolve(root, "runtime-events.ndjson");

  await mkdir(resolve(stateRoot, sessionId), { recursive: true });

  const lines = [
    {
      id: "1",
      timestamp: "2026-01-01T00:00:00.000Z",
      type: "session.started",
      severity: "info",
      message: "started",
      sessionId,
    },
    {
      id: "2",
      timestamp: "2026-01-01T00:00:01.000Z",
      type: "session.failed",
      severity: "critical",
      message: "Pipeline aborted after hard failures.",
      sessionId,
    },
  ];

  await writeFile(runtimeLogPath, `${lines.map((line) => JSON.stringify(line)).join("\n")}\n`, "utf8");

  const sessions = await buildSessionSummaries({
    stateRoot,
    runtimeEventLogPath: runtimeLogPath,
  });

  assert.equal(sessions.length, 1);
  assert.equal(sessions[0]?.status, "failure");
  assert.equal(sessions[0]?.aborted, true);
});