Initial commit: hardened DeerFlow factory

Vendored deer-flow upstream (bytedance/deer-flow) plus prompt-injection
hardening:

- New deerflow.security package: content_delimiter, html_cleaner,
  sanitizer (8 layers — invisible chars, control chars, symbols, NFC,
  PUA, tag chars, horizontal whitespace collapse with newline/tab
  preservation, length cap)
- New deerflow.community.searx package: web_search, web_fetch,
  image_search backed by a private SearX instance, every external
  string sanitized and wrapped in <<<EXTERNAL_UNTRUSTED_CONTENT>>>
  delimiters
- All native community web providers (ddg_search, tavily, exa,
  firecrawl, jina_ai, infoquest, image_search) replaced with hard-fail
  stubs that raise NativeWebToolDisabledError at import time, so a
  misconfigured tool.use path fails loud rather than silently falling
  back to unsanitized output
- Native client back-doors (jina_client.py, infoquest_client.py)
  stubbed too
- Native-tool tests quarantined under tests/_disabled_native/
  (collect_ignore_glob via local conftest.py)
- Sanitizer Layer 7 fix: only collapse horizontal whitespace, preserve
  newlines and tabs so list/table structure survives
- Hardened runtime config.yaml references only the searx-backed tools
- Factory overlay (backend/) kept in sync with deer-flow tree as a
  reference / source

See HARDENING.md for the full audit trail and verification steps.

deer-flow/backend/tests/test_clarification_middleware.py

@@ -0,0 +1,120 @@
+"""Tests for ClarificationMiddleware, focusing on options type coercion."""
+
+import json
+
+import pytest
+
+from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
+
+
+@pytest.fixture
+def middleware():
+    return ClarificationMiddleware()
+
+
+class TestFormatClarificationMessage:
+    """Tests for _format_clarification_message options handling."""
+
+    def test_options_as_native_list(self, middleware):
+        """Normal case: options is already a list."""
+        args = {
+            "question": "Which env?",
+            "clarification_type": "approach_choice",
+            "options": ["dev", "staging", "prod"],
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1. dev" in result
+        assert "2. staging" in result
+        assert "3. prod" in result
+
+    def test_options_as_json_string(self, middleware):
+        """Bug case (#1995): model serializes options as a JSON string."""
+        args = {
+            "question": "Which env?",
+            "clarification_type": "approach_choice",
+            "options": json.dumps(["dev", "staging", "prod"]),
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1. dev" in result
+        assert "2. staging" in result
+        assert "3. prod" in result
+        # Must NOT contain per-character output
+        assert "1. [" not in result
+        assert '2. "' not in result
+
+    def test_options_as_json_string_scalar(self, middleware):
+        """JSON string decoding to a non-list scalar is treated as one option."""
+        args = {
+            "question": "Which env?",
+            "clarification_type": "approach_choice",
+            "options": json.dumps("development"),
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1. development" in result
+        # Must be a single option, not per-character iteration.
+        assert "2." not in result
+
+    def test_options_as_plain_string(self, middleware):
+        """Edge case: options is a non-JSON string, treated as single option."""
+        args = {
+            "question": "Which env?",
+            "clarification_type": "approach_choice",
+            "options": "just one option",
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1. just one option" in result
+
+    def test_options_none(self, middleware):
+        """Options is None — no options section rendered."""
+        args = {
+            "question": "Tell me more",
+            "clarification_type": "missing_info",
+            "options": None,
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1." not in result
+
+    def test_options_empty_list(self, middleware):
+        """Options is an empty list — no options section rendered."""
+        args = {
+            "question": "Tell me more",
+            "clarification_type": "missing_info",
+            "options": [],
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1." not in result
+
+    def test_options_missing(self, middleware):
+        """Options key is absent — defaults to empty list."""
+        args = {
+            "question": "Tell me more",
+            "clarification_type": "missing_info",
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1." not in result
+
+    def test_context_included(self, middleware):
+        """Context is rendered before the question."""
+        args = {
+            "question": "Which env?",
+            "clarification_type": "approach_choice",
+            "context": "Need target env for config",
+            "options": ["dev", "prod"],
+        }
+        result = middleware._format_clarification_message(args)
+        assert "Need target env for config" in result
+        assert "Which env?" in result
+        assert "1. dev" in result
+
+    def test_json_string_with_mixed_types(self, middleware):
+        """JSON string containing non-string elements still works."""
+        args = {
+            "question": "Pick one",
+            "clarification_type": "approach_choice",
+            "options": json.dumps(["Option A", 2, True, None]),
+        }
+        result = middleware._format_clarification_message(args)
+        assert "1. Option A" in result
+        assert "2. 2" in result
+        assert "3. True" in result
+        assert "4. None" in result