first commit

This commit is contained in:
2026-02-25 23:49:54 -05:00
commit 4d097161cb
1775 changed files with 452827 additions and 0 deletions

575
tests/test_graph.py Normal file
View File

@@ -0,0 +1,575 @@
"""Tests for AppFactoryOrchestrator (LangGraph state machine)."""
import asyncio
import json
import os
import tempfile
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app_factory.core.graph import AppFactoryOrchestrator, AppFactoryState
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_pm_agent():
agent = AsyncMock()
agent.expand_prompt_to_prd = AsyncMock(return_value="# Generated PRD\n\nObjective: Build an app")
agent.handle_clarification_request = AsyncMock(return_value="Clarification: use REST API")
return agent
@pytest.fixture
def mock_task_agent():
agent = AsyncMock()
agent.parse_prd = AsyncMock(return_value={"tasks": []})
agent.get_unblocked_tasks = AsyncMock(return_value=[
{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []},
{"id": 2, "title": "Task 2", "status": "pending", "dependencies": []},
])
agent.update_task_status = AsyncMock()
agent.get_task_details = AsyncMock(return_value={"id": 1, "title": "Task 1"})
return agent
@pytest.fixture
def mock_dev_manager():
manager = AsyncMock()
manager.execute_with_retry = AsyncMock(return_value={
"status": "success",
"output": "Task completed",
"files_changed": ["app.py"],
"exit_code": 0,
})
return manager
@pytest.fixture
def mock_qa_agent():
agent = AsyncMock()
agent.max_retries = 3
agent.review_and_merge = AsyncMock(return_value={
"status": "merged",
"commit_sha": "abc123",
"review_summary": "LGTM",
})
return agent
@pytest.fixture
def mock_workspace_manager():
manager = AsyncMock()
manager.create_worktree = AsyncMock(return_value="/tmp/worktree/task-1")
manager.spin_up_clean_room = AsyncMock(return_value=MagicMock(id="container-123"))
manager.cleanup_workspace = AsyncMock()
return manager
@pytest.fixture
def mock_observability():
obs = MagicMock()
obs.log_state_transition = MagicMock()
obs.log_error = MagicMock()
return obs
@pytest.fixture
def orchestrator(mock_pm_agent, mock_task_agent, mock_dev_manager,
mock_qa_agent, mock_workspace_manager, mock_observability):
return AppFactoryOrchestrator(
pm_agent=mock_pm_agent,
task_agent=mock_task_agent,
dev_manager=mock_dev_manager,
qa_agent=mock_qa_agent,
workspace_manager=mock_workspace_manager,
observability=mock_observability,
)
# ---------------------------------------------------------------------------
# State Schema Tests
# ---------------------------------------------------------------------------
class TestStateSchema:
def test_state_has_all_required_fields(self):
required_fields = {
"user_input", "prd", "tasks", "active_tasks", "completed_tasks",
"blocked_tasks", "clarification_requests", "global_architecture",
"iteration_count", "max_iterations", "errors",
}
assert required_fields == set(AppFactoryState.__annotations__.keys())
def test_state_can_be_instantiated(self):
state: AppFactoryState = {
"user_input": "build a todo app",
"prd": "",
"tasks": [],
"active_tasks": {},
"completed_tasks": [],
"blocked_tasks": {},
"clarification_requests": [],
"global_architecture": "",
"iteration_count": 0,
"max_iterations": 50,
"errors": [],
}
assert state["user_input"] == "build a todo app"
assert state["max_iterations"] == 50
# ---------------------------------------------------------------------------
# Graph Construction Tests
# ---------------------------------------------------------------------------
class TestGraphBuild:
def test_graph_builds_without_errors(self):
orch = AppFactoryOrchestrator()
compiled = orch.build_graph()
assert compiled is not None
def test_graph_builds_with_all_agents(self, orchestrator):
compiled = orchestrator.build_graph()
assert compiled is not None
def test_graph_builds_with_none_agents(self):
orch = AppFactoryOrchestrator(
pm_agent=None, task_agent=None, dev_manager=None,
qa_agent=None, workspace_manager=None, observability=None,
)
compiled = orch.build_graph()
assert compiled is not None
# ---------------------------------------------------------------------------
# Node Tests
# ---------------------------------------------------------------------------
class TestPMNode:
@pytest.mark.asyncio
async def test_pm_node_sets_prd(self, orchestrator, mock_pm_agent):
state = {"user_input": "build a todo app", "errors": []}
result = await orchestrator._pm_node(state)
assert "prd" in result
assert result["prd"] == "# Generated PRD\n\nObjective: Build an app"
mock_pm_agent.expand_prompt_to_prd.assert_called_once_with("build a todo app")
@pytest.mark.asyncio
async def test_pm_node_handles_no_input(self, orchestrator):
state = {"user_input": "", "errors": []}
result = await orchestrator._pm_node(state)
assert result["prd"] == ""
assert any("No user input" in e for e in result["errors"])
@pytest.mark.asyncio
async def test_pm_node_handles_agent_error(self, orchestrator, mock_pm_agent):
mock_pm_agent.expand_prompt_to_prd.side_effect = RuntimeError("API down")
state = {"user_input": "build app", "errors": []}
result = await orchestrator._pm_node(state)
assert result["prd"] == ""
assert any("PM agent error" in e for e in result["errors"])
@pytest.mark.asyncio
async def test_pm_node_without_agent(self):
orch = AppFactoryOrchestrator()
state = {"user_input": "build a todo app", "errors": []}
result = await orch._pm_node(state)
assert "Mock PRD" in result["prd"]
class TestTaskNode:
@pytest.mark.asyncio
async def test_task_node_populates_tasks(self, orchestrator, mock_task_agent):
state = {"prd": "some PRD", "tasks": [], "iteration_count": 0, "max_iterations": 50, "errors": []}
result = await orchestrator._task_node(state)
assert "tasks" in result
assert len(result["tasks"]) == 2
mock_task_agent.parse_prd.assert_called_once()
@pytest.mark.asyncio
async def test_task_node_increments_iteration(self, orchestrator):
state = {"prd": "PRD", "tasks": [], "iteration_count": 5, "max_iterations": 50, "errors": []}
result = await orchestrator._task_node(state)
assert result["iteration_count"] == 6
@pytest.mark.asyncio
async def test_task_node_refreshes_on_subsequent_passes(self, orchestrator, mock_task_agent):
state = {
"prd": "PRD",
"tasks": [{"id": 1, "title": "existing", "status": "pending", "dependencies": []}],
"iteration_count": 1,
"max_iterations": 50,
"errors": [],
}
result = await orchestrator._task_node(state)
mock_task_agent.parse_prd.assert_not_called()
mock_task_agent.get_unblocked_tasks.assert_called()
@pytest.mark.asyncio
async def test_task_node_stops_at_max_iterations(self, orchestrator):
state = {"prd": "PRD", "tasks": [], "iteration_count": 49, "max_iterations": 50, "errors": []}
result = await orchestrator._task_node(state)
assert result["iteration_count"] == 50
assert any("Max iterations" in e for e in result["errors"])
class TestDevDispatchNode:
@pytest.mark.asyncio
async def test_dev_dispatch_spawns_concurrent_tasks(self, orchestrator, mock_dev_manager, mock_workspace_manager):
state = {
"tasks": [
{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []},
{"id": 2, "title": "Task 2", "status": "pending", "dependencies": []},
],
"completed_tasks": [],
"active_tasks": {},
"errors": [],
"clarification_requests": [],
"global_architecture": "",
}
result = await orchestrator._dev_dispatch_node(state)
assert "1" in result["completed_tasks"]
assert "2" in result["completed_tasks"]
assert mock_dev_manager.execute_with_retry.call_count == 2
assert mock_workspace_manager.create_worktree.call_count == 2
@pytest.mark.asyncio
async def test_dev_dispatch_handles_needs_clarification(self, orchestrator, mock_dev_manager):
mock_dev_manager.execute_with_retry.return_value = {
"status": "needs_clarification",
"output": "Cannot figure out API format",
"files_changed": [],
"exit_code": -1,
}
state = {
"tasks": [{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []}],
"completed_tasks": [],
"active_tasks": {},
"errors": [],
"clarification_requests": [],
"global_architecture": "",
}
result = await orchestrator._dev_dispatch_node(state)
assert len(result["clarification_requests"]) == 1
assert result["clarification_requests"][0]["task_id"] == "1"
@pytest.mark.asyncio
async def test_dev_dispatch_skips_completed(self, orchestrator):
state = {
"tasks": [{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []}],
"completed_tasks": ["1"],
"active_tasks": {},
"errors": [],
"clarification_requests": [],
"global_architecture": "",
}
result = await orchestrator._dev_dispatch_node(state)
assert result == {}
@pytest.mark.asyncio
async def test_dev_dispatch_without_agents(self):
orch = AppFactoryOrchestrator()
state = {
"tasks": [
{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []},
],
"completed_tasks": [],
"active_tasks": {},
"errors": [],
"clarification_requests": [],
"global_architecture": "",
}
result = await orch._dev_dispatch_node(state)
assert "1" in result["completed_tasks"]
class TestQANode:
@pytest.mark.asyncio
async def test_qa_node_processes_results(self, orchestrator, mock_qa_agent, mock_task_agent):
state = {
"tasks": [{"id": "1", "title": "Task 1"}],
"active_tasks": {"1": {"status": "success", "worktree_path": "/tmp/wt"}},
"completed_tasks": ["1"],
"errors": [],
"clarification_requests": [],
"blocked_tasks": {},
}
result = await orchestrator._qa_node(state)
mock_qa_agent.review_and_merge.assert_called_once()
assert result["active_tasks"]["1"]["status"] == "merged"
@pytest.mark.asyncio
async def test_qa_node_no_tasks_for_qa(self, orchestrator):
state = {
"tasks": [],
"active_tasks": {},
"completed_tasks": [],
"errors": [],
"clarification_requests": [],
"blocked_tasks": {},
}
result = await orchestrator._qa_node(state)
assert result == {}
@pytest.mark.asyncio
async def test_qa_node_handles_qa_failure(self, orchestrator, mock_qa_agent):
mock_qa_agent.review_and_merge.return_value = {
"status": "tests_failed",
"retry_count": 3,
}
state = {
"tasks": [{"id": "1", "title": "Task 1"}],
"active_tasks": {"1": {"status": "success", "worktree_path": "/tmp/wt"}},
"completed_tasks": ["1"],
"errors": [],
"clarification_requests": [],
"blocked_tasks": {},
}
result = await orchestrator._qa_node(state)
assert len(result["clarification_requests"]) == 1
# ---------------------------------------------------------------------------
# Routing Tests
# ---------------------------------------------------------------------------
class TestRoutingAfterTasks:
def test_routes_to_dev_dispatch_when_unblocked(self, orchestrator):
state = {
"tasks": [{"id": 1, "status": "pending", "dependencies": []}],
"completed_tasks": [],
"blocked_tasks": {},
"clarification_requests": [],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_tasks(state) == "dev_dispatch"
def test_routes_to_end_when_all_done(self, orchestrator):
state = {
"tasks": [{"id": 1, "status": "done", "dependencies": []}],
"completed_tasks": ["1"],
"blocked_tasks": {},
"clarification_requests": [],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_tasks(state) == "end"
def test_routes_to_clarification_when_blocked(self, orchestrator):
state = {
"tasks": [{"id": 1, "status": "pending", "dependencies": [2]}],
"completed_tasks": [],
"blocked_tasks": {"1": "dependency not met"},
"clarification_requests": [],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_tasks(state) == "clarification"
def test_routes_to_end_at_max_iterations(self, orchestrator):
state = {
"tasks": [{"id": 1, "status": "pending", "dependencies": []}],
"completed_tasks": [],
"blocked_tasks": {},
"clarification_requests": [],
"iteration_count": 50,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_tasks(state) == "end"
def test_routes_to_end_when_no_tasks(self, orchestrator):
state = {
"tasks": [],
"completed_tasks": [],
"blocked_tasks": {},
"clarification_requests": [],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_tasks(state) == "end"
class TestRoutingAfterQA:
def test_routes_to_task_node(self, orchestrator):
state = {
"clarification_requests": [],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_qa(state) == "task_node"
def test_routes_to_clarification(self, orchestrator):
state = {
"clarification_requests": [{"question": "What API?"}],
"iteration_count": 1,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_qa(state) == "clarification"
def test_routes_to_end_at_max_iterations(self, orchestrator):
state = {
"clarification_requests": [],
"iteration_count": 50,
"max_iterations": 50,
}
assert orchestrator._should_continue_after_qa(state) == "end"
# ---------------------------------------------------------------------------
# Iteration Safety Tests
# ---------------------------------------------------------------------------
class TestIterationSafety:
@pytest.mark.asyncio
async def test_iteration_limit_prevents_infinite_loop(self, orchestrator):
state = {
"prd": "PRD",
"tasks": [],
"iteration_count": 49,
"max_iterations": 50,
"errors": [],
}
result = await orchestrator._task_node(state)
assert result["iteration_count"] == 50
assert any("Max iterations" in e for e in result["errors"])
# ---------------------------------------------------------------------------
# State Persistence Tests
# ---------------------------------------------------------------------------
class TestStatePersistence:
def test_save_state(self, orchestrator):
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "state.json")
state = {
"user_input": "test",
"prd": "PRD content",
"tasks": [{"id": 1}],
"active_tasks": {},
"completed_tasks": ["1"],
"blocked_tasks": {},
"clarification_requests": [],
"global_architecture": "",
"iteration_count": 5,
"max_iterations": 50,
"errors": [],
}
orchestrator.save_state(state, path)
assert os.path.exists(path)
with open(path) as f:
loaded = json.load(f)
assert loaded["user_input"] == "test"
assert loaded["iteration_count"] == 5
def test_load_state(self, orchestrator):
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "state.json")
data = {"user_input": "hello", "prd": "PRD", "iteration_count": 3}
with open(path, "w") as f:
json.dump(data, f)
loaded = orchestrator.load_state(path)
assert loaded["user_input"] == "hello"
assert loaded["iteration_count"] == 3
def test_save_state_creates_directory(self, orchestrator):
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "nested", "dir", "state.json")
orchestrator.save_state({"user_input": "test"}, path)
assert os.path.exists(path)
# ---------------------------------------------------------------------------
# Clarification Node Tests
# ---------------------------------------------------------------------------
class TestClarificationNode:
@pytest.mark.asyncio
async def test_clarification_resolves_requests(self, orchestrator, mock_pm_agent):
state = {
"clarification_requests": [
{"requesting_agent": "dev", "task_id": "1", "question": "What format?", "context": ""},
],
"blocked_tasks": {"1": "needs clarification"},
"errors": [],
}
result = await orchestrator._clarification_node(state)
assert result["clarification_requests"] == []
assert "1" not in result["blocked_tasks"]
mock_pm_agent.handle_clarification_request.assert_called_once()
@pytest.mark.asyncio
async def test_clarification_handles_empty_requests(self, orchestrator):
state = {"clarification_requests": [], "blocked_tasks": {}, "errors": []}
result = await orchestrator._clarification_node(state)
assert result["clarification_requests"] == []
@pytest.mark.asyncio
async def test_clarification_without_agent(self):
orch = AppFactoryOrchestrator()
state = {
"clarification_requests": [{"task_id": "1", "question": "?"}],
"blocked_tasks": {"1": "blocked"},
"errors": [],
}
result = await orch._clarification_node(state)
assert result["clarification_requests"] == []
# ---------------------------------------------------------------------------
# End-to-End Run Test
# ---------------------------------------------------------------------------
class TestEndToEnd:
@pytest.mark.asyncio
async def test_run_executes_end_to_end(self, mock_pm_agent, mock_task_agent,
mock_dev_manager, mock_qa_agent,
mock_workspace_manager, mock_observability):
# After first get_unblocked_tasks returns tasks, the second call returns empty
# to terminate the loop.
mock_task_agent.get_unblocked_tasks = AsyncMock(
side_effect=[
[{"id": 1, "title": "Task 1", "status": "pending", "dependencies": []}],
[], # No more tasks - triggers end
]
)
orch = AppFactoryOrchestrator(
pm_agent=mock_pm_agent,
task_agent=mock_task_agent,
dev_manager=mock_dev_manager,
qa_agent=mock_qa_agent,
workspace_manager=mock_workspace_manager,
observability=mock_observability,
)
with tempfile.TemporaryDirectory() as tmpdir:
state_path = os.path.join(tmpdir, "state.json")
with patch.object(orch, "save_state") as mock_save:
result = await orch.run("build a todo app")
assert result["prd"] != ""
mock_pm_agent.expand_prompt_to_prd.assert_called_once_with("build a todo app")
mock_task_agent.parse_prd.assert_called_once()
@pytest.mark.asyncio
async def test_run_with_no_agents(self):
orch = AppFactoryOrchestrator()
with tempfile.TemporaryDirectory() as tmpdir:
state_path = os.path.join(tmpdir, "state.json")
with patch.object(orch, "save_state"):
result = await orch.run("build something")
assert "Mock PRD" in result["prd"]