diff --git a/automation/anwarpy/cover.py b/automation/anwarpy/cover.py new file mode 100644 index 0000000..e0f7446 --- /dev/null +++ b/automation/anwarpy/cover.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import base64 +import hashlib +from pathlib import Path +import random +import re + +from PIL import Image, ImageDraw, ImageFilter + +from publication_meta import CATEGORIES + +WIDTH = 1200 +HEIGHT = 630 +MAX_BYTES = 5 * 1024 * 1024 +SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + +_PALETTES = { + "وكلاء": ((17, 25, 48), (82, 214, 186)), + "نماذج": ((20, 22, 48), (129, 140, 248)), + "استرجاع": ((13, 31, 43), (56, 189, 248)), + "سلامة": ((37, 21, 35), (244, 114, 182)), + "أدوات": ((16, 30, 35), (245, 174, 65)), + "أعمال": ((31, 24, 20), (249, 115, 80)), +} + + +def _art(slug: str, category: str) -> Image.Image: + seed = int.from_bytes(hashlib.sha256(f"{slug}|{category}".encode()).digest()[:8], "big") + rng = random.Random(seed) + background, accent = _PALETTES[category] + image = Image.new("RGB", (WIDTH, HEIGHT), background) + + glow = Image.new("RGBA", image.size, (0, 0, 0, 0)) + glow_draw = ImageDraw.Draw(glow, "RGBA") + for _ in range(5): + x = rng.randint(80, WIDTH - 80) + y = rng.randint(60, HEIGHT - 60) + radius = rng.randint(130, 320) + glow_draw.ellipse( + (x - radius, y - radius, x + radius, y + radius), + fill=(*accent, rng.randint(18, 42)), + ) + glow = glow.filter(ImageFilter.GaussianBlur(70)) + image = Image.alpha_composite(image.convert("RGBA"), glow) + + overlay = Image.new("RGBA", image.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay, "RGBA") + points = [(rng.randint(110, WIDTH - 110), rng.randint(90, HEIGHT - 90)) for _ in range(12)] + for index, start in enumerate(points): + end = points[(index * 5 + 3) % len(points)] + draw.line((start, end), fill=(*accent, 95), width=3) + for index, (x, y) in enumerate(points): + radius = 8 + (index % 4) * 4 + draw.ellipse( + (x - radius, y - radius, x + radius, y + radius), + fill=(*accent, 220), + outline=(235, 241, 247, 180), + width=2, + ) + + left = 68 + top = 54 + right = WIDTH - 68 + bottom = HEIGHT - 54 + draw.rounded_rectangle((left, top, right, bottom), radius=32, outline=(*accent, 120), width=3) + draw.line((left + 35, bottom - 55, right - 35, bottom - 55), fill=(*accent, 155), width=4) + return Image.alpha_composite(image, overlay).convert("RGB") + + +def generate_cover(slug: str, category: str, output_dir: Path) -> dict: + if not SLUG_RE.fullmatch(slug): + raise ValueError("slug must be lowercase ASCII kebab-case") + if category not in CATEGORIES: + raise ValueError("category is invalid") + + output_dir.mkdir(parents=True, exist_ok=True) + output = output_dir / f"{slug}.jpg" + temporary = output.with_suffix(".jpg.tmp") + image = _art(slug, category) + image.save(temporary, format="JPEG", quality=91, optimize=True, progressive=True) + + if temporary.stat().st_size > MAX_BYTES: + temporary.unlink(missing_ok=True) + raise ValueError("generated cover exceeds 5MB") + with Image.open(temporary) as check: + if check.size != (WIDTH, HEIGHT) or check.format != "JPEG" or check.mode != "RGB": + temporary.unlink(missing_ok=True) + raise ValueError("generated cover failed validation") + temporary.replace(output) + + encoded = base64.b64encode(output.read_bytes()).decode("ascii") + return { + "path": output, + "coverImage": {"filename": output.name, "dataBase64": encoded}, + } diff --git a/automation/anwarpy/publish_one.py b/automation/anwarpy/publish_one.py index 9302dc2..92974e8 100644 --- a/automation/anwarpy/publish_one.py +++ b/automation/anwarpy/publish_one.py @@ -227,8 +227,8 @@ class Publisher: meta.get("slug") if meta["status"] == "republish" else None, ) media = self.media.render(article, meta) - if not isinstance(media, dict) or "coverImage" not in media or "video_path" not in media: - raise DeterministicFailure("cover", "media output is incomplete") + if not isinstance(media, dict) or "coverImage" not in media: + raise DeterministicFailure("cover", "cover output is incomplete") except DeterministicFailure as error: return self._block(candidate, error.reason) except ValueError: diff --git a/automation/anwarpy/runtime.py b/automation/anwarpy/runtime.py new file mode 100644 index 0000000..9b5d050 --- /dev/null +++ b/automation/anwarpy/runtime.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path +import time +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +from cover import generate_cover +from publish_one import AmbiguousWrite, DeterministicFailure, Publisher, _validate_article + +WORKSPACE_ID = "6082aea5-a29e-410f-97be-3cd55fbc2a3a" +ROOT_VIEW_ID = "3a743541-abbf-495a-a18a-74f405fa5d70" + + +def _json_from_text(text: str) -> Any: + value: Any = json.loads(text) + while isinstance(value, dict) and isinstance(value.get("result"), str): + value = json.loads(value["result"]) + return value + + +class AppFlowyMcp: + """Root-scoped AppFlowy access through Hermes's configured MCP endpoint.""" + + def __init__(self, config_path: Path | None = None): + self.config_path = config_path or Path.home() / ".hermes/config.yaml" + self.allowed_page_ids: set[str] = set() + + async def _call_async(self, tool: str, arguments: dict[str, Any]) -> Any: + import httpx + import yaml + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + + config = yaml.safe_load(self.config_path.read_text())["mcp_servers"]["appflowy"] + timeout = httpx.Timeout(30, read=120) + async with httpx.AsyncClient( + headers=config.get("headers", {}), + follow_redirects=True, + timeout=timeout, + ) as http: + async with streamable_http_client(config["url"], http_client=http) as streams: # type: ignore[arg-type] + async with ClientSession(streams[0], streams[1]) as session: + await session.initialize() + result = await session.call_tool(tool, arguments) + if getattr(result, "is_error", False): + raise RuntimeError(f"AppFlowy MCP call failed: {tool}") + if not result.content or not hasattr(result.content[0], "text"): + raise RuntimeError(f"AppFlowy MCP returned no JSON: {tool}") + return _json_from_text(getattr(result.content[0], "text")) + + def _call(self, tool: str, arguments: dict[str, Any]) -> Any: + # ponytail: one session per call; keep one open only if scheduled scans become measurably slow. + return asyncio.run(self._call_async(tool, arguments)) + + @staticmethod + def _flatten(children: list[dict[str, Any]]) -> list[dict[str, Any]]: + pages: list[dict[str, Any]] = [] + for child in children: + pages.append(child) + pages.extend(AppFlowyMcp._flatten(child.get("children") or [])) + return pages + + def list_pages(self) -> list[dict[str, Any]]: + response = self._call( + "get_workspace_folder", + {"workspace_id": WORKSPACE_ID, "root_view_id": ROOT_VIEW_ID, "depth": 10}, + ) + root = response.get("data", response) + if root.get("view_id") != ROOT_VIEW_ID: + raise RuntimeError("AppFlowy returned the wrong source root") + pages = self._flatten(root.get("children") or []) + self.allowed_page_ids = {page["view_id"] for page in pages} + return pages + + def _require_allowed(self, page_id: str) -> None: + if page_id not in self.allowed_page_ids: + raise PermissionError("page is outside the configured Ai research subtree") + + def get_blocks(self, page_id: str) -> list[dict[str, Any]]: + self._require_allowed(page_id) + response = self._call( + "get_page_blocks", + {"workspace_id": WORKSPACE_ID, "page_id": page_id}, + ) + return response.get("blocks") or response.get("data", {}).get("blocks") or [] + + def edit_block(self, page_id: str, block_id: str, text: str) -> None: + self._require_allowed(page_id) + self._call( + "edit_block_text", + { + "workspace_id": WORKSPACE_ID, + "page_id": page_id, + "block_id": block_id, + "text": text, + }, + ) + + +class AgentApi: + def __init__(self, key: str, base_url: str = "https://anwarpy.ai"): + if not key: + raise ValueError("ANWARPY_AGENT_KEY is required") + self.key = key + self.base_url = base_url.rstrip("/") + self.body_probes: dict[str, str] = {} + + def _request(self, method: str, path: str, body: dict[str, Any] | None = None, *, anonymous: bool = False): + data = None if body is None else json.dumps(body, ensure_ascii=False).encode() + headers = {"Accept": "application/json", "User-Agent": "anwarpy-publisher/1"} + if body is not None: + headers["Content-Type"] = "application/json" + if not anonymous: + headers["x-agent-key"] = self.key + request = Request(f"{self.base_url}{path}", data=data, headers=headers, method=method) + try: + with urlopen(request, timeout=30) as response: + raw = response.read() + return response.status, raw, response.headers.get("content-type", "") + except HTTPError as error: + return error.code, error.read(), error.headers.get("content-type", "") + except (TimeoutError, URLError) as error: + if method in ("POST", "PUT"): + raise AmbiguousWrite(f"agent API {method} outcome is unknown") from error + raise + + @staticmethod + def _decode(status: int, raw: bytes) -> dict[str, Any]: + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"agent API returned invalid JSON ({status})") from error + if not isinstance(value, dict): + raise RuntimeError(f"agent API returned invalid object ({status})") + return value + + @staticmethod + def _probe(body: dict[str, Any]) -> str | None: + children = body.get("content", {}).get("root", {}).get("children", []) + for child in children: + for text in child.get("children", []): + value = text.get("text") + if isinstance(value, str) and len(value.strip()) >= 20: + return value.strip() + return None + + def lookup(self, slug: str) -> dict[str, Any] | None: + status, raw, _ = self._request("GET", f"/api/agent/posts/{quote(slug)}") + if status == 404: + return None + if status != 200: + raise RuntimeError(f"agent API lookup failed ({status})") + return self._decode(status, raw) + + def create(self, body: dict[str, Any]) -> dict[str, Any]: + probe = self._probe(body) + if probe: + self.body_probes[body["slug"]] = probe + status, raw, _ = self._request("POST", "/api/agent/posts", body) + if status == 409: + existing = self.lookup(body["slug"]) + if existing is None: + raise RuntimeError("agent API returned 409 without an existing slug") + return existing + if status != 201: + raise RuntimeError(f"agent API create failed ({status})") + return self._decode(status, raw) + + def update(self, slug: str, body: dict[str, Any]) -> dict[str, Any]: + probe = self._probe(body) + if probe: + self.body_probes[slug] = probe + status, raw, _ = self._request("PUT", f"/api/agent/posts/{quote(slug)}", body) + if status != 200: + raise RuntimeError(f"agent API update failed ({status})") + return self._decode(status, raw) + + def _public_get(self, url: str, *, rsc: bool = False) -> tuple[int, bytes]: + headers = {"Accept": "text/x-component" if rsc else "text/html", "User-Agent": "anwarpy-publisher/1"} + if rsc: + headers["RSC"] = "1" + request = Request(url, headers=headers) + try: + with urlopen(request, timeout=20) as response: + return response.status, response.read() + except HTTPError as error: + return error.code, error.read() + + def verify_public(self, record: dict[str, Any]) -> bool: + slug = record["slug"] + canonical = f"https://anwarpy.ai/posts/{slug}" + if record.get("url", canonical) != canonical: + return False + + probe = self.body_probes.get(slug, "").encode() + for attempt in range(5): + html_status, html = self._public_get(canonical) + rsc_status, rsc = self._public_get(f"{canonical}?__rsc=publisher", rsc=True) + rest_status, rest, _ = self._request( + "GET", + f"/api/posts?where%5Bslug%5D%5Bequals%5D={quote(slug)}&depth=0", + anonymous=True, + ) + status_ok = html_status == 200 and rsc_status in (200, 204) and rest_status in (200, 401, 403) + vip_leak = record.get("access") == "vip" and probe and any(probe in value for value in (html, rsc, rest)) + if status_ok and not vip_leak: + return True + if attempt < 4: + time.sleep(2) + return False + + +class LocalArticleGenerator: + def __init__(self, endpoint: str = "http://100.64.0.5:8080/v1/chat/completions", model: str | None = None): + self.endpoint = endpoint + self.model = model + + def _model_id(self) -> str: + if self.model: + return self.model + models_url = self.endpoint.rsplit("/chat/completions", 1)[0] + "/models" + try: + with urlopen(models_url, timeout=10) as response: + data = json.loads(response.read()) + model = data["data"][0]["id"] + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError, KeyError, IndexError) as error: + raise DeterministicFailure("generation", "local model discovery failed") from error + if not isinstance(model, str) or not model: + raise DeterministicFailure("generation", "local model discovery returned no model") + self.model = model + return model + + def _request(self, messages: list[dict[str, str]]) -> str: + payload = { + "model": self._model_id(), + "messages": messages, + "temperature": 0.35, + "max_tokens": 7000, + "response_format": {"type": "json_object"}, + } + request = Request( + self.endpoint, + data=json.dumps(payload, ensure_ascii=False).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=240) as response: + data = json.loads(response.read()) + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error: + raise DeterministicFailure("generation", "local article model failed") from error + return data["choices"][0]["message"]["content"] + + @staticmethod + def _parse(raw: str, access: str, expected_slug: str | None) -> dict[str, Any]: + try: + article = json.loads(raw) + except json.JSONDecodeError as error: + raise DeterministicFailure("article_schema", "model returned invalid JSON") from error + article = _validate_article(article, access, expected_slug) + visible_words = sum( + len(paragraph.split()) + for section in article["sections"] + for paragraph in section["paragraphs"] + ) + if not 900 <= visible_words <= 2500: + raise DeterministicFailure("article_schema", "article must contain 900 to 2500 visible words") + return article + + def generate(self, title: str, evidence: str, meta: dict[str, str]) -> dict[str, Any]: + expected_slug = meta.get("slug") if meta["status"] == "republish" else None + prompt = f"""اكتب دليلاً عربياً أصلياً لموقع anwarpy.ai اعتماداً حصراً على الأدلة أدناه. +العنوان المصدر: {title} +نوع الوصول الذي حدده المالك: {meta['access']} +التصنيف الذي حدده المالك: {meta['category']} + +أعد JSON فقط بهذه الحقول: title, slug, dek, teaser, sections. +slug إنجليزي ASCII بصيغة kebab-case، ولا تغيّره عند إعادة النشر: {expected_slug or 'اختر وصفاً دقيقاً'}. +sections مصفوفة من 3 إلى 6 عناصر؛ كل عنصر يحتوي heading وparagraphs وcode. +النص المرئي بين 900 و2500 كلمة. رتّب الدليل: المشكلة، المتطلبات، التنفيذ، التحقق، القيود. +لا تختر أو تغيّر access أو category. لا تخترع إصدارات أو أرقاماً أو نتائج أو اقتباسات. +مقال VIP يحتاج teaser عربي مفيداً من فقرتين، والمقال المجاني يستخدم teaser فارغاً. + +الأدلة المنقحة: +{evidence} +""" + messages = [ + {"role": "system", "content": "أنت محرر تقني عربي دقيق. أعد JSON صالحاً فقط."}, + {"role": "user", "content": prompt}, + ] + raw = self._request(messages) + try: + return self._parse(raw, meta["access"], expected_slug) + except DeterministicFailure as first_error: + corrected = self._request(messages + [ + {"role": "assistant", "content": raw}, + {"role": "user", "content": f"صحح JSON فقط. خطأ التحقق: {first_error}"}, + ]) + return self._parse(corrected, meta["access"], expected_slug) + + +class CoverMedia: + def __init__(self, output_dir: Path | None = None): + self.output_dir = output_dir or Path.home() / ".hermes/cache/scratch/anwarpy-covers" + + def render(self, article: dict[str, Any], meta: dict[str, str]) -> dict[str, Any]: + try: + return generate_cover(article["slug"], meta["category"], self.output_dir) + except (OSError, ValueError) as error: + raise DeterministicFailure("cover", "cover generation failed") from error + + +def main() -> int: + key = os.environ.get("ANWARPY_AGENT_KEY", "") + publisher = Publisher( + appflowy=AppFlowyMcp(), + api=AgentApi(key), + generator=LocalArticleGenerator(), + media=CoverMedia(), + lock_path=Path.home() / ".hermes/locks/anwarpy-publish.lock", + ) + result = publisher.run() + print(result) + for error in publisher.marker_errors: + print(f"marker-error: {error}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/automation/anwarpy/test_cover.py b/automation/anwarpy/test_cover.py new file mode 100644 index 0000000..89a169b --- /dev/null +++ b/automation/anwarpy/test_cover.py @@ -0,0 +1,36 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from cover import generate_cover + + +class CoverTests(unittest.TestCase): + def test_generates_deterministic_1200x630_jpeg_under_5mb(self): + output_dir = Path(tempfile.mkdtemp()) + first = generate_cover("arabic-ai-guide", "أدوات", output_dir) + first_hash = hashlib.sha256(first["path"].read_bytes()).hexdigest() + second = generate_cover("arabic-ai-guide", "أدوات", output_dir) + second_hash = hashlib.sha256(second["path"].read_bytes()).hexdigest() + + self.assertEqual(first_hash, second_hash) + self.assertEqual(first["coverImage"]["filename"], "arabic-ai-guide.jpg") + self.assertLessEqual(first["path"].stat().st_size, 5 * 1024 * 1024) + with Image.open(first["path"]) as image: + self.assertEqual(image.size, (1200, 630)) + self.assertEqual(image.format, "JPEG") + self.assertEqual(image.mode, "RGB") + + def test_rejects_invalid_slug_or_category(self): + output_dir = Path(tempfile.mkdtemp()) + with self.assertRaises(ValueError): + generate_cover("عنوان عربي", "أدوات", output_dir) + with self.assertRaises(ValueError): + generate_cover("valid-slug", "عام", output_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/automation/anwarpy/test_runtime.py b/automation/anwarpy/test_runtime.py new file mode 100644 index 0000000..a45f3b1 --- /dev/null +++ b/automation/anwarpy/test_runtime.py @@ -0,0 +1,118 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from runtime import ( + AgentApi, + AppFlowyMcp, + CoverMedia, + LocalArticleGenerator, + _json_from_text, +) + + +class FakeAppFlowy(AppFlowyMcp): + def __init__(self): + super().__init__(Path("unused")) + self.edits = [] + + def _call(self, tool, arguments): + if tool == "get_workspace_folder": + return { + "data": { + "view_id": "3a743541-abbf-495a-a18a-74f405fa5d70", + "children": [ + {"view_id": "a", "name": "A", "children": [ + {"view_id": "b", "name": "B", "children": []}, + ]}, + ], + } + } + if tool == "get_page_blocks": + return {"blocks": [{"id": "x", "text": "body"}]} + if tool == "edit_block_text": + self.edits.append(arguments) + return {"code": 0} + raise AssertionError(tool) + + +class FakeAgentApi(AgentApi): + def __init__(self): + super().__init__("test-key") + self.responses = [] + self.public = (200, b"page") + self.rsc = (200, b"rsc") + + def _request(self, method, path, body=None, *, anonymous=False): + if self.responses: + return self.responses.pop(0) + return 200, b"{}", "application/json" + + def _public_get(self, url, *, rsc=False): + return self.rsc if rsc else self.public + + +class RuntimeTests(unittest.TestCase): + def test_unwraps_nested_mcp_result(self): + nested = json.dumps({"result": json.dumps({"data": {"ok": True}})}) + self.assertEqual(_json_from_text(nested), {"data": {"ok": True}}) + + def test_appflowy_enforces_enumerated_subtree(self): + client = FakeAppFlowy() + self.assertEqual([page["view_id"] for page in client.list_pages()], ["a", "b"]) + self.assertEqual(client.get_blocks("b")[0]["id"], "x") + client.edit_block("a", "x", "new") + self.assertEqual(client.edits[0]["page_id"], "a") + with self.assertRaises(PermissionError): + client.get_blocks("outside") + + def test_agent_api_records_body_probe_and_decodes_create(self): + api = FakeAgentApi() + api.responses = [(201, json.dumps({"slug": "guide"}).encode(), "application/json")] + body = { + "slug": "guide", + "content": {"root": {"children": [ + {"children": [{"text": "distinctive protected body sentence"}]}, + ]}}, + } + self.assertEqual(api.create(body)["slug"], "guide") + self.assertEqual(api.body_probes["guide"], "distinctive protected body sentence") + + @patch("runtime.time.sleep", return_value=None) + def test_vip_verification_rejects_body_leak(self, _sleep): + api = FakeAgentApi() + api.body_probes["guide"] = "VIP_BODY_MUST_NOT_LEAK" + api.public = (200, b"VIP_BODY_MUST_NOT_LEAK") + record = {"slug": "guide", "access": "vip", "url": "https://anwarpy.ai/posts/guide"} + self.assertFalse(api.verify_public(record)) + + def test_local_article_parser_enforces_word_count(self): + article = { + "title": "دليل عربي", + "slug": "arabic-guide", + "dek": "وصف عملي", + "teaser": "", + "sections": [ + {"heading": "أ", "paragraphs": ["كلمة " * 300], "code": None}, + {"heading": "ب", "paragraphs": ["كلمة " * 300], "code": None}, + {"heading": "ج", "paragraphs": ["كلمة " * 300], "code": None}, + ], + } + parsed = LocalArticleGenerator._parse(json.dumps(article, ensure_ascii=False), "free", None) + self.assertEqual(parsed["slug"], "arabic-guide") + article["sections"][0]["paragraphs"] = ["قصير"] + with self.assertRaisesRegex(Exception, "900 to 2500"): + LocalArticleGenerator._parse(json.dumps(article, ensure_ascii=False), "free", None) + + def test_cover_media_returns_cover_without_video(self): + media = CoverMedia(Path(tempfile.mkdtemp())).render( + {"slug": "arabic-guide"}, {"category": "أدوات"} + ) + self.assertIn("coverImage", media) + self.assertNotIn("video_path", media) + + +if __name__ == "__main__": + unittest.main()