337 lines
14 KiB
Python
337 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
from typing import Any
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import quote
|
|
from urllib.request import Request, urlopen
|
|
|
|
from cover import generate_cover
|
|
from publish_one import AmbiguousWrite, DeterministicFailure, Publisher, _validate_article
|
|
|
|
WORKSPACE_ID = "6082aea5-a29e-410f-97be-3cd55fbc2a3a"
|
|
ROOT_VIEW_ID = "3a743541-abbf-495a-a18a-74f405fa5d70"
|
|
|
|
|
|
def _json_from_text(text: str) -> Any:
|
|
value: Any = json.loads(text)
|
|
while isinstance(value, dict) and isinstance(value.get("result"), str):
|
|
value = json.loads(value["result"])
|
|
return value
|
|
|
|
|
|
class AppFlowyMcp:
|
|
"""Root-scoped AppFlowy access through Hermes's configured MCP endpoint."""
|
|
|
|
def __init__(self, config_path: Path | None = None):
|
|
self.config_path = config_path or Path.home() / ".hermes/config.yaml"
|
|
self.allowed_page_ids: set[str] = set()
|
|
|
|
async def _call_async(self, tool: str, arguments: dict[str, Any]) -> Any:
|
|
import httpx
|
|
import yaml
|
|
from mcp import ClientSession
|
|
from mcp.client.streamable_http import streamable_http_client
|
|
|
|
config = yaml.safe_load(self.config_path.read_text())["mcp_servers"]["appflowy"]
|
|
timeout = httpx.Timeout(30, read=120)
|
|
async with httpx.AsyncClient(
|
|
headers=config.get("headers", {}),
|
|
follow_redirects=True,
|
|
timeout=timeout,
|
|
) as http:
|
|
async with streamable_http_client(config["url"], http_client=http) as streams: # type: ignore[arg-type]
|
|
async with ClientSession(streams[0], streams[1]) as session:
|
|
await session.initialize()
|
|
result = await session.call_tool(tool, arguments)
|
|
if getattr(result, "is_error", False):
|
|
raise RuntimeError(f"AppFlowy MCP call failed: {tool}")
|
|
if not result.content or not hasattr(result.content[0], "text"):
|
|
raise RuntimeError(f"AppFlowy MCP returned no JSON: {tool}")
|
|
return _json_from_text(getattr(result.content[0], "text"))
|
|
|
|
def _call(self, tool: str, arguments: dict[str, Any]) -> Any:
|
|
# ponytail: one session per call; keep one open only if scheduled scans become measurably slow.
|
|
return asyncio.run(self._call_async(tool, arguments))
|
|
|
|
@staticmethod
|
|
def _flatten(children: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
pages: list[dict[str, Any]] = []
|
|
for child in children:
|
|
pages.append(child)
|
|
pages.extend(AppFlowyMcp._flatten(child.get("children") or []))
|
|
return pages
|
|
|
|
def list_pages(self) -> list[dict[str, Any]]:
|
|
response = self._call(
|
|
"get_workspace_folder",
|
|
{"workspace_id": WORKSPACE_ID, "root_view_id": ROOT_VIEW_ID, "depth": 10},
|
|
)
|
|
root = response.get("data", response)
|
|
if root.get("view_id") != ROOT_VIEW_ID:
|
|
raise RuntimeError("AppFlowy returned the wrong source root")
|
|
pages = self._flatten(root.get("children") or [])
|
|
self.allowed_page_ids = {page["view_id"] for page in pages}
|
|
return pages
|
|
|
|
def _require_allowed(self, page_id: str) -> None:
|
|
if page_id not in self.allowed_page_ids:
|
|
raise PermissionError("page is outside the configured Ai research subtree")
|
|
|
|
def get_blocks(self, page_id: str) -> list[dict[str, Any]]:
|
|
self._require_allowed(page_id)
|
|
response = self._call(
|
|
"get_page_blocks",
|
|
{"workspace_id": WORKSPACE_ID, "page_id": page_id},
|
|
)
|
|
return response.get("blocks") or response.get("data", {}).get("blocks") or []
|
|
|
|
def edit_block(self, page_id: str, block_id: str, text: str) -> None:
|
|
self._require_allowed(page_id)
|
|
self._call(
|
|
"edit_block_text",
|
|
{
|
|
"workspace_id": WORKSPACE_ID,
|
|
"page_id": page_id,
|
|
"block_id": block_id,
|
|
"text": text,
|
|
},
|
|
)
|
|
|
|
|
|
class AgentApi:
|
|
def __init__(self, key: str, base_url: str = "https://anwarpy.ai"):
|
|
if not key:
|
|
raise ValueError("ANWARPY_AGENT_KEY is required")
|
|
self.key = key
|
|
self.base_url = base_url.rstrip("/")
|
|
self.body_probes: dict[str, str] = {}
|
|
|
|
def _request(self, method: str, path: str, body: dict[str, Any] | None = None, *, anonymous: bool = False):
|
|
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
|
headers = {"Accept": "application/json", "User-Agent": "anwarpy-publisher/1"}
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
if not anonymous:
|
|
headers["x-agent-key"] = self.key
|
|
request = Request(f"{self.base_url}{path}", data=data, headers=headers, method=method)
|
|
try:
|
|
with urlopen(request, timeout=30) as response:
|
|
raw = response.read()
|
|
return response.status, raw, response.headers.get("content-type", "")
|
|
except HTTPError as error:
|
|
return error.code, error.read(), error.headers.get("content-type", "")
|
|
except (TimeoutError, URLError) as error:
|
|
if method in ("POST", "PATCH"):
|
|
raise AmbiguousWrite(f"agent API {method} outcome is unknown") from error
|
|
raise
|
|
|
|
@staticmethod
|
|
def _decode(status: int, raw: bytes) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(raw)
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
raise RuntimeError(f"agent API returned invalid JSON ({status})") from error
|
|
if not isinstance(value, dict):
|
|
raise RuntimeError(f"agent API returned invalid object ({status})")
|
|
return value
|
|
|
|
@staticmethod
|
|
def _probe(body: dict[str, Any]) -> str | None:
|
|
children = body.get("content", {}).get("root", {}).get("children", [])
|
|
for child in children:
|
|
for text in child.get("children", []):
|
|
value = text.get("text")
|
|
if isinstance(value, str) and len(value.strip()) >= 20:
|
|
return value.strip()
|
|
return None
|
|
|
|
def lookup(self, slug: str) -> dict[str, Any] | None:
|
|
status, raw, _ = self._request("GET", f"/api/agent/posts/{quote(slug)}")
|
|
if status == 404:
|
|
return None
|
|
if status != 200:
|
|
raise RuntimeError(f"agent API lookup failed ({status})")
|
|
return self._decode(status, raw)
|
|
|
|
def create(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
probe = self._probe(body)
|
|
if probe:
|
|
self.body_probes[body["slug"]] = probe
|
|
status, raw, _ = self._request("POST", "/api/agent/posts", body)
|
|
if status == 409:
|
|
existing = self.lookup(body["slug"])
|
|
if existing is None:
|
|
raise RuntimeError("agent API returned 409 without an existing slug")
|
|
return existing
|
|
if status != 201:
|
|
raise RuntimeError(f"agent API create failed ({status})")
|
|
return self._decode(status, raw)
|
|
|
|
def update(self, slug: str, body: dict[str, Any]) -> dict[str, Any]:
|
|
probe = self._probe(body)
|
|
if probe:
|
|
self.body_probes[slug] = probe
|
|
status, raw, _ = self._request("PATCH", f"/api/agent/posts/{quote(slug)}", body)
|
|
if status != 200:
|
|
raise RuntimeError(f"agent API update failed ({status})")
|
|
return self._decode(status, raw)
|
|
|
|
def _public_get(self, url: str, *, rsc: bool = False) -> tuple[int, bytes]:
|
|
headers = {"Accept": "text/x-component" if rsc else "text/html", "User-Agent": "anwarpy-publisher/1"}
|
|
if rsc:
|
|
headers["RSC"] = "1"
|
|
request = Request(url, headers=headers)
|
|
try:
|
|
with urlopen(request, timeout=20) as response:
|
|
return response.status, response.read()
|
|
except HTTPError as error:
|
|
return error.code, error.read()
|
|
|
|
def verify_public(self, record: dict[str, Any]) -> bool:
|
|
slug = record["slug"]
|
|
canonical = f"https://anwarpy.ai/posts/{slug}"
|
|
if record.get("url", canonical) != canonical:
|
|
return False
|
|
|
|
probe = self.body_probes.get(slug, "").encode()
|
|
for attempt in range(5):
|
|
html_status, html = self._public_get(canonical)
|
|
rsc_status, rsc = self._public_get(f"{canonical}?__rsc=publisher", rsc=True)
|
|
rest_status, rest, _ = self._request(
|
|
"GET",
|
|
f"/api/posts?where%5Bslug%5D%5Bequals%5D={quote(slug)}&depth=0",
|
|
anonymous=True,
|
|
)
|
|
status_ok = html_status == 200 and rsc_status in (200, 204) and rest_status in (200, 401, 403)
|
|
vip_leak = record.get("access") == "vip" and probe and any(probe in value for value in (html, rsc, rest))
|
|
if status_ok and not vip_leak:
|
|
return True
|
|
if attempt < 4:
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
class LocalArticleGenerator:
|
|
def __init__(self, endpoint: str = "http://100.64.0.5:8080/v1/chat/completions", model: str | None = None):
|
|
self.endpoint = endpoint
|
|
self.model = model
|
|
|
|
def _model_id(self) -> str:
|
|
if self.model:
|
|
return self.model
|
|
models_url = self.endpoint.rsplit("/chat/completions", 1)[0] + "/models"
|
|
try:
|
|
with urlopen(models_url, timeout=10) as response:
|
|
data = json.loads(response.read())
|
|
model = data["data"][0]["id"]
|
|
except (HTTPError, URLError, TimeoutError, json.JSONDecodeError, KeyError, IndexError) as error:
|
|
raise DeterministicFailure("generation", "local model discovery failed") from error
|
|
if not isinstance(model, str) or not model:
|
|
raise DeterministicFailure("generation", "local model discovery returned no model")
|
|
self.model = model
|
|
return model
|
|
|
|
def _request(self, messages: list[dict[str, str]]) -> str:
|
|
payload = {
|
|
"model": self._model_id(),
|
|
"messages": messages,
|
|
"temperature": 0.35,
|
|
"max_tokens": 7000,
|
|
"response_format": {"type": "json_object"},
|
|
}
|
|
request = Request(
|
|
self.endpoint,
|
|
data=json.dumps(payload, ensure_ascii=False).encode(),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=240) as response:
|
|
data = json.loads(response.read())
|
|
except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error:
|
|
raise DeterministicFailure("generation", "local article model failed") from error
|
|
return data["choices"][0]["message"]["content"]
|
|
|
|
@staticmethod
|
|
def _parse(raw: str, access: str, expected_slug: str | None) -> dict[str, Any]:
|
|
try:
|
|
article = json.loads(raw)
|
|
except json.JSONDecodeError as error:
|
|
raise DeterministicFailure("article_schema", "model returned invalid JSON") from error
|
|
article = _validate_article(article, access, expected_slug)
|
|
visible_words = sum(
|
|
len(paragraph.split())
|
|
for section in article["sections"]
|
|
for paragraph in section["paragraphs"]
|
|
)
|
|
if not 900 <= visible_words <= 2500:
|
|
raise DeterministicFailure("article_schema", "article must contain 900 to 2500 visible words")
|
|
return article
|
|
|
|
def generate(self, title: str, evidence: str, meta: dict[str, str]) -> dict[str, Any]:
|
|
expected_slug = meta.get("slug") if meta["status"] == "republish" else None
|
|
prompt = f"""اكتب دليلاً عربياً أصلياً لموقع anwarpy.ai اعتماداً حصراً على الأدلة أدناه.
|
|
العنوان المصدر: {title}
|
|
نوع الوصول الذي حدده المالك: {meta['access']}
|
|
التصنيف الذي حدده المالك: {meta['category']}
|
|
|
|
أعد JSON فقط بهذه الحقول: title, slug, dek, teaser, sections.
|
|
slug إنجليزي ASCII بصيغة kebab-case، ولا تغيّره عند إعادة النشر: {expected_slug or 'اختر وصفاً دقيقاً'}.
|
|
sections مصفوفة من 3 إلى 6 عناصر؛ كل عنصر يحتوي heading وparagraphs وcode.
|
|
النص المرئي بين 900 و2500 كلمة. رتّب الدليل: المشكلة، المتطلبات، التنفيذ، التحقق، القيود.
|
|
لا تختر أو تغيّر access أو category. لا تخترع إصدارات أو أرقاماً أو نتائج أو اقتباسات.
|
|
مقال VIP يحتاج teaser عربي مفيداً من فقرتين، والمقال المجاني يستخدم teaser فارغاً.
|
|
|
|
الأدلة المنقحة:
|
|
{evidence}
|
|
"""
|
|
messages = [
|
|
{"role": "system", "content": "أنت محرر تقني عربي دقيق. أعد JSON صالحاً فقط."},
|
|
{"role": "user", "content": prompt},
|
|
]
|
|
raw = self._request(messages)
|
|
try:
|
|
return self._parse(raw, meta["access"], expected_slug)
|
|
except DeterministicFailure as first_error:
|
|
corrected = self._request(messages + [
|
|
{"role": "assistant", "content": raw},
|
|
{"role": "user", "content": f"صحح JSON فقط. خطأ التحقق: {first_error}"},
|
|
])
|
|
return self._parse(corrected, meta["access"], expected_slug)
|
|
|
|
|
|
class CoverMedia:
|
|
def __init__(self, output_dir: Path | None = None):
|
|
self.output_dir = output_dir or Path.home() / ".hermes/cache/scratch/anwarpy-covers"
|
|
|
|
def render(self, article: dict[str, Any], meta: dict[str, str]) -> dict[str, Any]:
|
|
try:
|
|
return generate_cover(article["slug"], meta["category"], self.output_dir)
|
|
except (OSError, ValueError) as error:
|
|
raise DeterministicFailure("cover", "cover generation failed") from error
|
|
|
|
|
|
def main() -> int:
|
|
key = os.environ.get("ANWARPY_AGENT_KEY", "")
|
|
publisher = Publisher(
|
|
appflowy=AppFlowyMcp(),
|
|
api=AgentApi(key),
|
|
generator=LocalArticleGenerator(),
|
|
media=CoverMedia(),
|
|
lock_path=Path.home() / ".hermes/locks/anwarpy-publish.lock",
|
|
)
|
|
result = publisher.run()
|
|
print(result)
|
|
for error in publisher.marker_errors:
|
|
print(f"marker-error: {error}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|