Initial commit: hardened DeerFlow factory
Vendored deer-flow upstream (bytedance/deer-flow) plus prompt-injection hardening: - New deerflow.security package: content_delimiter, html_cleaner, sanitizer (8 layers — invisible chars, control chars, symbols, NFC, PUA, tag chars, horizontal whitespace collapse with newline/tab preservation, length cap) - New deerflow.community.searx package: web_search, web_fetch, image_search backed by a private SearX instance, every external string sanitized and wrapped in <<<EXTERNAL_UNTRUSTED_CONTENT>>> delimiters - All native community web providers (ddg_search, tavily, exa, firecrawl, jina_ai, infoquest, image_search) replaced with hard-fail stubs that raise NativeWebToolDisabledError at import time, so a misconfigured tool.use path fails loud rather than silently falling back to unsanitized output - Native client back-doors (jina_client.py, infoquest_client.py) stubbed too - Native-tool tests quarantined under tests/_disabled_native/ (collect_ignore_glob via local conftest.py) - Sanitizer Layer 7 fix: only collapse horizontal whitespace, preserve newlines and tabs so list/table structure survives - Hardened runtime config.yaml references only the searx-backed tools - Factory overlay (backend/) kept in sync with deer-flow tree as a reference / source See HARDENING.md for the full audit trail and verification steps.
This commit is contained in:
330
deer-flow/backend/tests/test_client_live.py
Normal file
330
deer-flow/backend/tests/test_client_live.py
Normal file
@@ -0,0 +1,330 @@
|
||||
"""Live integration tests for DeerFlowClient with real API.
|
||||
|
||||
These tests require a working config.yaml with valid API credentials.
|
||||
They are skipped in CI and must be run explicitly:
|
||||
|
||||
PYTHONPATH=. uv run pytest tests/test_client_live.py -v -s
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.client import DeerFlowClient, StreamEvent
|
||||
from deerflow.sandbox.security import is_host_bash_allowed
|
||||
from deerflow.uploads.manager import PathTraversalError
|
||||
|
||||
# Skip entire module in CI or when no config.yaml exists
|
||||
_skip_reason = None
|
||||
if os.environ.get("CI"):
|
||||
_skip_reason = "Live tests skipped in CI"
|
||||
elif not Path(__file__).resolve().parents[2].joinpath("config.yaml").exists():
|
||||
_skip_reason = "No config.yaml found — live tests require valid API credentials"
|
||||
|
||||
if _skip_reason:
|
||||
pytest.skip(_skip_reason, allow_module_level=True)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Create a real DeerFlowClient (no mocks)."""
|
||||
return DeerFlowClient(thinking_enabled=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def thread_tmp(tmp_path):
|
||||
"""Provide a unique thread_id + tmp directory for file operations."""
|
||||
import uuid
|
||||
|
||||
tid = f"live-test-{uuid.uuid4().hex[:8]}"
|
||||
return tid, tmp_path
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 1: Basic chat — model responds coherently
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveBasicChat:
|
||||
def test_chat_returns_nonempty_string(self, client):
|
||||
"""chat() returns a non-empty response from the real model."""
|
||||
response = client.chat("Reply with exactly: HELLO")
|
||||
assert isinstance(response, str)
|
||||
assert len(response) > 0
|
||||
print(f" chat response: {response}")
|
||||
|
||||
def test_chat_follows_instruction(self, client):
|
||||
"""Model can follow a simple instruction."""
|
||||
response = client.chat("What is 7 * 8? Reply with just the number.")
|
||||
assert "56" in response
|
||||
print(f" math response: {response}")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 2: Streaming — events arrive in correct order
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveStreaming:
|
||||
def test_stream_yields_messages_tuple_and_end(self, client):
|
||||
"""stream() produces at least one messages-tuple event and ends with end."""
|
||||
events = list(client.stream("Say hi in one word."))
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert "messages-tuple" in types, f"Expected 'messages-tuple' event, got: {types}"
|
||||
assert "values" in types, f"Expected 'values' event, got: {types}"
|
||||
assert types[-1] == "end"
|
||||
|
||||
for e in events:
|
||||
assert isinstance(e, StreamEvent)
|
||||
print(f" [{e.type}] {e.data}")
|
||||
|
||||
def test_stream_ai_content_nonempty(self, client):
|
||||
"""Streamed messages-tuple AI events contain non-empty content."""
|
||||
ai_messages = [e for e in client.stream("What color is the sky? One word.") if e.type == "messages-tuple" and e.data.get("type") == "ai" and e.data.get("content")]
|
||||
assert len(ai_messages) >= 1
|
||||
for m in ai_messages:
|
||||
assert len(m.data.get("content", "")) > 0
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 3: Tool use — agent calls a tool and returns result
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveToolUse:
|
||||
def test_agent_uses_bash_tool(self, client):
|
||||
"""Agent uses bash tool when asked to run a command."""
|
||||
if not is_host_bash_allowed():
|
||||
pytest.skip("Host bash is disabled for LocalSandboxProvider in the active config")
|
||||
|
||||
events = list(client.stream("Use the bash tool to run: echo 'LIVE_TEST_OK'. Then tell me the output."))
|
||||
|
||||
types = [e.type for e in events]
|
||||
print(f" event types: {types}")
|
||||
for e in events:
|
||||
print(f" [{e.type}] {e.data}")
|
||||
|
||||
# All message events are now messages-tuple
|
||||
mt_events = [e for e in events if e.type == "messages-tuple"]
|
||||
tc_events = [e for e in mt_events if e.data.get("type") == "ai" and "tool_calls" in e.data]
|
||||
tr_events = [e for e in mt_events if e.data.get("type") == "tool"]
|
||||
ai_events = [e for e in mt_events if e.data.get("type") == "ai" and e.data.get("content")]
|
||||
|
||||
assert len(tc_events) >= 1, f"Expected tool_call event, got types: {types}"
|
||||
assert len(tr_events) >= 1, f"Expected tool result event, got types: {types}"
|
||||
assert len(ai_events) >= 1
|
||||
|
||||
assert tc_events[0].data["tool_calls"][0]["name"] == "bash"
|
||||
assert "LIVE_TEST_OK" in tr_events[0].data["content"]
|
||||
|
||||
def test_agent_uses_ls_tool(self, client):
|
||||
"""Agent uses ls tool to list a directory."""
|
||||
events = list(client.stream("Use the ls tool to list the contents of /mnt/user-data/workspace. Just report what you see."))
|
||||
|
||||
types = [e.type for e in events]
|
||||
print(f" event types: {types}")
|
||||
|
||||
tc_events = [e for e in events if e.type == "messages-tuple" and e.data.get("type") == "ai" and "tool_calls" in e.data]
|
||||
assert len(tc_events) >= 1
|
||||
assert tc_events[0].data["tool_calls"][0]["name"] == "ls"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 4: Multi-tool chain — agent chains tools in sequence
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveMultiToolChain:
|
||||
def test_write_then_read(self, client):
|
||||
"""Agent writes a file, then reads it back."""
|
||||
events = list(client.stream("Step 1: Use write_file to write 'integration_test_content' to /mnt/user-data/outputs/live_test.txt. Step 2: Use read_file to read that file back. Step 3: Tell me the content you read."))
|
||||
|
||||
types = [e.type for e in events]
|
||||
print(f" event types: {types}")
|
||||
for e in events:
|
||||
print(f" [{e.type}] {e.data}")
|
||||
|
||||
tc_events = [e for e in events if e.type == "messages-tuple" and e.data.get("type") == "ai" and "tool_calls" in e.data]
|
||||
tool_names = [tc.data["tool_calls"][0]["name"] for tc in tc_events]
|
||||
|
||||
assert "write_file" in tool_names, f"Expected write_file, got: {tool_names}"
|
||||
assert "read_file" in tool_names, f"Expected read_file, got: {tool_names}"
|
||||
|
||||
# Final AI message or tool result should mention the content
|
||||
ai_events = [e for e in events if e.type == "messages-tuple" and e.data.get("type") == "ai" and e.data.get("content")]
|
||||
tr_events = [e for e in events if e.type == "messages-tuple" and e.data.get("type") == "tool"]
|
||||
final_text = ai_events[-1].data["content"] if ai_events else ""
|
||||
assert "integration_test_content" in final_text.lower() or any("integration_test_content" in e.data.get("content", "") for e in tr_events)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 5: File upload lifecycle with real filesystem
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveFileUpload:
|
||||
def test_upload_list_delete(self, client, thread_tmp):
|
||||
"""Upload → list → delete → verify deletion."""
|
||||
thread_id, tmp_path = thread_tmp
|
||||
|
||||
# Create test files
|
||||
f1 = tmp_path / "test_upload_a.txt"
|
||||
f1.write_text("content A")
|
||||
f2 = tmp_path / "test_upload_b.txt"
|
||||
f2.write_text("content B")
|
||||
|
||||
# Upload
|
||||
result = client.upload_files(thread_id, [f1, f2])
|
||||
assert result["success"] is True
|
||||
assert len(result["files"]) == 2
|
||||
filenames = {r["filename"] for r in result["files"]}
|
||||
assert filenames == {"test_upload_a.txt", "test_upload_b.txt"}
|
||||
for r in result["files"]:
|
||||
assert int(r["size"]) > 0
|
||||
assert r["virtual_path"].startswith("/mnt/user-data/uploads/")
|
||||
assert "artifact_url" in r
|
||||
print(f" uploaded: {filenames}")
|
||||
|
||||
# List
|
||||
listed = client.list_uploads(thread_id)
|
||||
assert listed["count"] == 2
|
||||
print(f" listed: {[f['filename'] for f in listed['files']]}")
|
||||
|
||||
# Delete one
|
||||
del_result = client.delete_upload(thread_id, "test_upload_a.txt")
|
||||
assert del_result["success"] is True
|
||||
remaining = client.list_uploads(thread_id)
|
||||
assert remaining["count"] == 1
|
||||
assert remaining["files"][0]["filename"] == "test_upload_b.txt"
|
||||
print(f" after delete: {[f['filename'] for f in remaining['files']]}")
|
||||
|
||||
# Delete the other
|
||||
client.delete_upload(thread_id, "test_upload_b.txt")
|
||||
empty = client.list_uploads(thread_id)
|
||||
assert empty["count"] == 0
|
||||
assert empty["files"] == []
|
||||
|
||||
def test_upload_nonexistent_file_raises(self, client):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
client.upload_files("t-fail", ["/nonexistent/path/file.txt"])
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 6: Configuration query — real config loading
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveConfigQueries:
|
||||
def test_list_models_returns_configured_model(self, client):
|
||||
"""list_models() returns at least one configured model with Gateway-aligned fields."""
|
||||
result = client.list_models()
|
||||
assert "models" in result
|
||||
assert len(result["models"]) >= 1
|
||||
names = [m["name"] for m in result["models"]]
|
||||
# Verify Gateway-aligned fields
|
||||
for m in result["models"]:
|
||||
assert "display_name" in m
|
||||
assert "supports_thinking" in m
|
||||
print(f" models: {names}")
|
||||
|
||||
def test_get_model_found(self, client):
|
||||
"""get_model() returns details for the first configured model."""
|
||||
result = client.list_models()
|
||||
first_model_name = result["models"][0]["name"]
|
||||
model = client.get_model(first_model_name)
|
||||
assert model is not None
|
||||
assert model["name"] == first_model_name
|
||||
assert "display_name" in model
|
||||
assert "supports_thinking" in model
|
||||
print(f" model detail: {model}")
|
||||
|
||||
def test_get_model_not_found(self, client):
|
||||
assert client.get_model("nonexistent-model-xyz") is None
|
||||
|
||||
def test_list_skills(self, client):
|
||||
"""list_skills() runs without error."""
|
||||
result = client.list_skills()
|
||||
assert "skills" in result
|
||||
assert isinstance(result["skills"], list)
|
||||
print(f" skills count: {len(result['skills'])}")
|
||||
for s in result["skills"][:3]:
|
||||
print(f" - {s['name']}: {s['enabled']}")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 7: Artifact read after agent writes
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveArtifact:
|
||||
def test_get_artifact_after_write(self, client):
|
||||
"""Agent writes a file → client reads it back via get_artifact()."""
|
||||
import uuid
|
||||
|
||||
thread_id = f"live-artifact-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
# Ask agent to write a file
|
||||
events = list(
|
||||
client.stream(
|
||||
'Use write_file to create /mnt/user-data/outputs/artifact_test.json with content: {"status": "ok", "source": "live_test"}',
|
||||
thread_id=thread_id,
|
||||
)
|
||||
)
|
||||
|
||||
# Verify write happened
|
||||
tc_events = [e for e in events if e.type == "messages-tuple" and e.data.get("type") == "ai" and "tool_calls" in e.data]
|
||||
assert any(any(tc["name"] == "write_file" for tc in e.data["tool_calls"]) for e in tc_events)
|
||||
|
||||
# Read artifact
|
||||
content, mime = client.get_artifact(thread_id, "mnt/user-data/outputs/artifact_test.json")
|
||||
data = json.loads(content)
|
||||
assert data["status"] == "ok"
|
||||
assert data["source"] == "live_test"
|
||||
assert "json" in mime
|
||||
print(f" artifact: {data}, mime: {mime}")
|
||||
|
||||
def test_get_artifact_not_found(self, client):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
client.get_artifact("nonexistent-thread", "mnt/user-data/outputs/nope.txt")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 8: Per-call overrides
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveOverrides:
|
||||
def test_thinking_disabled_still_works(self, client):
|
||||
"""Explicit thinking_enabled=False override produces a response."""
|
||||
response = client.chat(
|
||||
"Say OK.",
|
||||
thinking_enabled=False,
|
||||
)
|
||||
assert len(response) > 0
|
||||
print(f" response: {response}")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Scenario 9: Error resilience
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestLiveErrorResilience:
|
||||
def test_delete_nonexistent_upload(self, client):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
client.delete_upload("nonexistent-thread", "ghost.txt")
|
||||
|
||||
def test_bad_artifact_path(self, client):
|
||||
with pytest.raises(ValueError):
|
||||
client.get_artifact("t", "invalid/path")
|
||||
|
||||
def test_path_traversal_blocked(self, client):
|
||||
with pytest.raises(PathTraversalError):
|
||||
client.delete_upload("t", "../../etc/passwd")
|
||||
Reference in New Issue
Block a user