Initial commit: hardened DeerFlow factory

Vendored deer-flow upstream (bytedance/deer-flow) plus prompt-injection hardening: - New deerflow.security package: content_delimiter, html_cleaner, sanitizer (8 layers — invisible chars, control chars, symbols, NFC, PUA, tag chars, horizontal whitespace collapse with newline/tab preservation, length cap) - New deerflow.community.searx package: web_search, web_fetch, image_search backed by a private SearX instance, every external string sanitized and wrapped in <<<EXTERNAL_UNTRUSTED_CONTENT>>> delimiters - All native community web providers (ddg_search, tavily, exa, firecrawl, jina_ai, infoquest, image_search) replaced with hard-fail stubs that raise NativeWebToolDisabledError at import time, so a misconfigured tool.use path fails loud rather than silently falling back to unsanitized output - Native client back-doors (jina_client.py, infoquest_client.py) stubbed too - Native-tool tests quarantined under tests/_disabled_native/ (collect_ignore_glob via local conftest.py) - Sanitizer Layer 7 fix: only collapse horizontal whitespace, preserve newlines and tabs so list/table structure survives - Hardened runtime config.yaml references only the searx-backed tools - Factory overlay (backend/) kept in sync with deer-flow tree as a reference / source See HARDENING.md for the full audit trail and verification steps.
2026-04-12 14:23:57 +02:00
commit 6de0bf9f5b
889 changed files with 173052 additions and 0 deletions
--- a/deer-flow/backend/tests/test_lead_agent_prompt.py
+++ b/deer-flow/backend/tests/test_lead_agent_prompt.py
@@ -0,0 +1,165 @@
+import threading
+from types import SimpleNamespace
+
+import anyio
+
+from deerflow.agents.lead_agent import prompt as prompt_module
+from deerflow.skills.types import Skill
+
+
+def test_build_custom_mounts_section_returns_empty_when_no_mounts(monkeypatch):
+    config = SimpleNamespace(sandbox=SimpleNamespace(mounts=[]))
+    monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
+
+    assert prompt_module._build_custom_mounts_section() == ""
+
+
+def test_build_custom_mounts_section_lists_configured_mounts(monkeypatch):
+    mounts = [
+        SimpleNamespace(container_path="/home/user/shared", read_only=False),
+        SimpleNamespace(container_path="/mnt/reference", read_only=True),
+    ]
+    config = SimpleNamespace(sandbox=SimpleNamespace(mounts=mounts))
+    monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
+
+    section = prompt_module._build_custom_mounts_section()
+
+    assert "**Custom Mounted Directories:**" in section
+    assert "`/home/user/shared`" in section
+    assert "read-write" in section
+    assert "`/mnt/reference`" in section
+    assert "read-only" in section
+
+
+def test_apply_prompt_template_includes_custom_mounts(monkeypatch):
+    mounts = [SimpleNamespace(container_path="/home/user/shared", read_only=False)]
+    config = SimpleNamespace(
+        sandbox=SimpleNamespace(mounts=mounts),
+        skills=SimpleNamespace(container_path="/mnt/skills"),
+    )
+    monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
+    monkeypatch.setattr(prompt_module, "_get_enabled_skills", lambda: [])
+    monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda: "")
+    monkeypatch.setattr(prompt_module, "_build_acp_section", lambda: "")
+    monkeypatch.setattr(prompt_module, "_get_memory_context", lambda agent_name=None: "")
+    monkeypatch.setattr(prompt_module, "get_agent_soul", lambda agent_name=None: "")
+
+    prompt = prompt_module.apply_prompt_template()
+
+    assert "`/home/user/shared`" in prompt
+    assert "Custom Mounted Directories" in prompt
+
+
+def test_apply_prompt_template_includes_relative_path_guidance(monkeypatch):
+    config = SimpleNamespace(
+        sandbox=SimpleNamespace(mounts=[]),
+        skills=SimpleNamespace(container_path="/mnt/skills"),
+    )
+    monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
+    monkeypatch.setattr(prompt_module, "_get_enabled_skills", lambda: [])
+    monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda: "")
+    monkeypatch.setattr(prompt_module, "_build_acp_section", lambda: "")
+    monkeypatch.setattr(prompt_module, "_get_memory_context", lambda agent_name=None: "")
+    monkeypatch.setattr(prompt_module, "get_agent_soul", lambda agent_name=None: "")
+
+    prompt = prompt_module.apply_prompt_template()
+
+    assert "Treat `/mnt/user-data/workspace` as your default current working directory" in prompt
+    assert "`hello.txt`, `../uploads/data.csv`, and `../outputs/report.md`" in prompt
+
+
+def test_refresh_skills_system_prompt_cache_async_reloads_immediately(monkeypatch, tmp_path):
+    def make_skill(name: str) -> Skill:
+        skill_dir = tmp_path / name
+        return Skill(
+            name=name,
+            description=f"Description for {name}",
+            license="MIT",
+            skill_dir=skill_dir,
+            skill_file=skill_dir / "SKILL.md",
+            relative_path=skill_dir.relative_to(tmp_path),
+            category="custom",
+            enabled=True,
+        )
+
+    state = {"skills": [make_skill("first-skill")]}
+    monkeypatch.setattr(prompt_module, "load_skills", lambda enabled_only=True: list(state["skills"]))
+    prompt_module._reset_skills_system_prompt_cache_state()
+
+    try:
+        prompt_module.warm_enabled_skills_cache()
+        assert [skill.name for skill in prompt_module._get_enabled_skills()] == ["first-skill"]
+
+        state["skills"] = [make_skill("second-skill")]
+        anyio.run(prompt_module.refresh_skills_system_prompt_cache_async)
+
+        assert [skill.name for skill in prompt_module._get_enabled_skills()] == ["second-skill"]
+    finally:
+        prompt_module._reset_skills_system_prompt_cache_state()
+
+
+def test_clear_cache_does_not_spawn_parallel_refresh_workers(monkeypatch, tmp_path):
+    started = threading.Event()
+    release = threading.Event()
+    active_loads = 0
+    max_active_loads = 0
+    call_count = 0
+    lock = threading.Lock()
+
+    def make_skill(name: str) -> Skill:
+        skill_dir = tmp_path / name
+        return Skill(
+            name=name,
+            description=f"Description for {name}",
+            license="MIT",
+            skill_dir=skill_dir,
+            skill_file=skill_dir / "SKILL.md",
+            relative_path=skill_dir.relative_to(tmp_path),
+            category="custom",
+            enabled=True,
+        )
+
+    def fake_load_skills(enabled_only=True):
+        nonlocal active_loads, max_active_loads, call_count
+        with lock:
+            active_loads += 1
+            max_active_loads = max(max_active_loads, active_loads)
+            call_count += 1
+            current_call = call_count
+
+        started.set()
+        if current_call == 1:
+            release.wait(timeout=5)
+
+        with lock:
+            active_loads -= 1
+
+        return [make_skill(f"skill-{current_call}")]
+
+    monkeypatch.setattr(prompt_module, "load_skills", fake_load_skills)
+    prompt_module._reset_skills_system_prompt_cache_state()
+
+    try:
+        prompt_module.clear_skills_system_prompt_cache()
+        assert started.wait(timeout=5)
+
+        prompt_module.clear_skills_system_prompt_cache()
+        release.set()
+        prompt_module.warm_enabled_skills_cache()
+
+        assert max_active_loads == 1
+        assert [skill.name for skill in prompt_module._get_enabled_skills()] == ["skill-2"]
+    finally:
+        release.set()
+        prompt_module._reset_skills_system_prompt_cache_state()
+
+
+def test_warm_enabled_skills_cache_logs_on_timeout(monkeypatch, caplog):
+    event = threading.Event()
+    monkeypatch.setattr(prompt_module, "_ensure_enabled_skills_cache", lambda: event)
+
+    with caplog.at_level("WARNING"):
+        warmed = prompt_module.warm_enabled_skills_cache(timeout_seconds=0.01)
+
+    assert warmed is False
+    assert "Timed out waiting" in caplog.text