feat: validate Anwarpy publication metadata
This commit is contained in:
AHMED
@@ -12,3 +12,7 @@ screenshot-*.png
|
||||
.DS_Store
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Python runtime files
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import Iterable, Mapping, Optional
|
||||
|
||||
from publication_meta import clean, parse_meta
|
||||
|
||||
|
||||
class MarkerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PageMarker:
|
||||
block_id: str
|
||||
raw: str
|
||||
meta: dict[str, str]
|
||||
|
||||
|
||||
def find_marker(blocks: Iterable[Mapping[str, object]]) -> Optional[PageMarker]:
|
||||
valid: list[PageMarker] = []
|
||||
near_misses: list[str] = []
|
||||
|
||||
for block in blocks:
|
||||
block_id = str(block.get("id", "unknown"))
|
||||
raw_value = block.get("text", "")
|
||||
raw = raw_value if isinstance(raw_value, str) else ""
|
||||
normalized = clean(raw)
|
||||
if re.search(r"anwarpy", normalized, re.IGNORECASE) is None:
|
||||
continue
|
||||
if not normalized.startswith("anwarpy:"):
|
||||
near_misses.append(block_id)
|
||||
continue
|
||||
try:
|
||||
meta = parse_meta(normalized)
|
||||
except ValueError:
|
||||
near_misses.append(block_id)
|
||||
continue
|
||||
valid.append(PageMarker(block_id=block_id, raw=raw, meta=meta))
|
||||
|
||||
if len(valid) > 1:
|
||||
ids = ", ".join(marker.block_id for marker in valid)
|
||||
raise MarkerError(f"multiple anwarpy markers found in blocks: {ids}")
|
||||
if valid and near_misses:
|
||||
ids = ", ".join(near_misses)
|
||||
raise MarkerError(f"ambiguous anwarpy markers; near-miss blocks: {ids}")
|
||||
if near_misses:
|
||||
ids = ", ".join(near_misses)
|
||||
raise MarkerError(f"near-miss anwarpy marker in blocks: {ids}")
|
||||
return valid[0] if valid else None
|
||||
@@ -0,0 +1 @@
|
||||
anwarpy.ai
|
||||
@@ -0,0 +1,155 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
CATEGORIES = ("وكلاء", "نماذج", "استرجاع", "سلامة", "أدوات", "أعمال")
|
||||
ACCESSES = ("free", "vip")
|
||||
STATUSES = ("pending", "publishing", "published", "blocked", "republish")
|
||||
BLOCKED_REASONS = ("sanitizer", "generation", "article_schema", "cover", "video")
|
||||
SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
_INVISIBLE = re.compile(r"[\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069\ufeff]")
|
||||
_NO_BREAK = re.compile(r"[\u00a0\u202f]")
|
||||
|
||||
_FIELD_ORDER = (
|
||||
"status",
|
||||
"access",
|
||||
"category",
|
||||
"queued_at",
|
||||
"slug",
|
||||
"published_at",
|
||||
"url",
|
||||
"blocked_reason",
|
||||
)
|
||||
_REQUIRED = {
|
||||
"pending": {"status", "access", "category", "queued_at"},
|
||||
"publishing": {"status", "access", "category", "queued_at", "slug"},
|
||||
"published": {"status", "access", "category", "queued_at", "slug", "published_at", "url"},
|
||||
"blocked": {"status", "access", "category", "queued_at", "blocked_reason"},
|
||||
"republish": {"status", "access", "category", "queued_at", "slug", "published_at", "url"},
|
||||
}
|
||||
|
||||
|
||||
def clean(line: str) -> str:
|
||||
line = _INVISIBLE.sub(" ", line)
|
||||
line = _NO_BREAK.sub(" ", line)
|
||||
return unicodedata.normalize("NFC", line).strip()
|
||||
|
||||
|
||||
def _aware_datetime(value: str, field: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{field} must be a valid ISO-8601 datetime") from exc
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise ValueError(f"{field} must include a timezone offset")
|
||||
return parsed
|
||||
|
||||
|
||||
def _validate_url(value: str, slug: str) -> None:
|
||||
parsed = urlsplit(value)
|
||||
expected_path = f"/posts/{slug}"
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.netloc != "anwarpy.ai"
|
||||
or parsed.path != expected_path
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ValueError(f"url must equal https://anwarpy.ai{expected_path}")
|
||||
|
||||
|
||||
def parse_meta(raw: str) -> dict[str, str]:
|
||||
line = clean(raw)
|
||||
if not line.startswith("anwarpy:"):
|
||||
raise ValueError("marker must start with exact prefix anwarpy:")
|
||||
|
||||
body = line.removeprefix("anwarpy:").strip()
|
||||
if not body:
|
||||
raise ValueError("marker fields are required")
|
||||
|
||||
meta: dict[str, str] = {}
|
||||
for part in body.split("|"):
|
||||
segment = clean(part)
|
||||
if "=" not in segment:
|
||||
raise ValueError(f"invalid marker field: {segment}")
|
||||
key, value = (clean(item) for item in segment.split("=", 1))
|
||||
if key not in _FIELD_ORDER:
|
||||
raise ValueError(f"unknown marker field: {key}")
|
||||
if key in meta:
|
||||
raise ValueError(f"duplicate marker field: {key}")
|
||||
if not value:
|
||||
raise ValueError(f"empty marker field: {key}")
|
||||
meta[key] = value
|
||||
|
||||
status = meta.get("status")
|
||||
if status not in STATUSES:
|
||||
raise ValueError("status is invalid")
|
||||
required = _REQUIRED[status]
|
||||
missing = required - meta.keys()
|
||||
if missing:
|
||||
raise ValueError(f"missing required fields: {', '.join(sorted(missing))}")
|
||||
forbidden = meta.keys() - required
|
||||
if forbidden:
|
||||
raise ValueError(f"fields not allowed for {status}: {', '.join(sorted(forbidden))}")
|
||||
|
||||
if meta["access"] not in ACCESSES:
|
||||
raise ValueError("access is invalid")
|
||||
if meta["category"] not in CATEGORIES:
|
||||
raise ValueError("category is invalid")
|
||||
_aware_datetime(meta["queued_at"], "queued_at")
|
||||
|
||||
slug = meta.get("slug")
|
||||
if slug is not None and not SLUG_RE.fullmatch(slug):
|
||||
raise ValueError("slug must be lowercase ASCII kebab-case")
|
||||
if "published_at" in meta:
|
||||
_aware_datetime(meta["published_at"], "published_at")
|
||||
if "url" in meta:
|
||||
_validate_url(meta["url"], meta["slug"])
|
||||
if "blocked_reason" in meta and meta["blocked_reason"] not in BLOCKED_REASONS:
|
||||
raise ValueError("blocked_reason is invalid")
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
def render_meta(meta: dict[str, str]) -> str:
|
||||
validated = parse_meta(
|
||||
"anwarpy: " + " | ".join(f"{key}={value}" for key, value in meta.items())
|
||||
)
|
||||
return "anwarpy: " + " | ".join(
|
||||
f"{key}={validated[key]}" for key in _FIELD_ORDER if key in validated
|
||||
)
|
||||
|
||||
|
||||
def same_meta(expected_raw: str, actual_raw: str) -> bool:
|
||||
return parse_meta(expected_raw) == parse_meta(actual_raw)
|
||||
|
||||
|
||||
def mark_publishing(raw: str, slug: str) -> str:
|
||||
meta = parse_meta(raw)
|
||||
if meta["status"] != "pending":
|
||||
raise ValueError("only pending markers can be claimed")
|
||||
meta["status"] = "publishing"
|
||||
meta["slug"] = clean(slug)
|
||||
return render_meta(meta)
|
||||
|
||||
|
||||
def mark_published(raw: str, published_at: str, url: str) -> str:
|
||||
meta = parse_meta(raw)
|
||||
if meta["status"] not in ("publishing", "republish"):
|
||||
raise ValueError("only publishing or republish markers can become published")
|
||||
meta["status"] = "published"
|
||||
meta["published_at"] = clean(published_at)
|
||||
meta["url"] = clean(url)
|
||||
return render_meta(meta)
|
||||
|
||||
|
||||
def mark_blocked(raw: str, reason: str) -> str:
|
||||
meta = parse_meta(raw)
|
||||
if meta["status"] != "pending":
|
||||
raise ValueError("only pending markers can be blocked")
|
||||
meta["status"] = "blocked"
|
||||
meta["blocked_reason"] = clean(reason)
|
||||
return render_meta(meta)
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
from typing import Iterable
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
REDACTION = "[REDACTED]"
|
||||
|
||||
_URL_RE = re.compile(r"https?://[^\s<>\]\[\)\(\"']+", re.IGNORECASE)
|
||||
_EMAIL_RE = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
|
||||
_IPV4_RE = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?![\d.])")
|
||||
_IPV6_CANDIDATE_RE = re.compile(r"(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{0,4}:){2,}[0-9A-Fa-f]{0,4}(?![0-9A-Fa-f:])")
|
||||
_PHONE_RE = re.compile(r"(?<!\w)\+?\d(?:[\s().-]*\d){7,14}(?!\w)")
|
||||
_PATH_RE = re.compile(r"(?:/(?:Users|home|root|opt|etc|var)/[^\s,;]+|[A-Za-z]:\\Users\\[^\s,;]+)")
|
||||
_CREDENTIAL_RE = re.compile(
|
||||
r"(?i)\b(?:password|passwd|secret|token|api[_-]?key|authorization)\s*[:=]\s*[^\s,;]+"
|
||||
)
|
||||
_PRIVATE_HOST_RE = re.compile(r"\b(?:[A-Z0-9-]+\.)+(?:local|internal|lan|home)\b", re.IGNORECASE)
|
||||
_FQDN_RE = re.compile(r"\b(?:[A-Z0-9-]+\.)+[A-Z]{2,63}\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def _normalized_hosts(hosts: Iterable[str]) -> set[str]:
|
||||
return {host.strip().lower().rstrip(".") for host in hosts if host.strip()}
|
||||
|
||||
|
||||
def _redact_url(match: re.Match[str], allowed_hosts: set[str]) -> str:
|
||||
value = match.group(0)
|
||||
host = (urlsplit(value).hostname or "").lower().rstrip(".")
|
||||
return value if host in allowed_hosts else REDACTION
|
||||
|
||||
|
||||
def _redact_ipv4(match: re.Match[str]) -> str:
|
||||
value = match.group(0)
|
||||
try:
|
||||
ipaddress.IPv4Address(value)
|
||||
except ipaddress.AddressValueError:
|
||||
return value
|
||||
return REDACTION
|
||||
|
||||
|
||||
def _redact_ipv6(match: re.Match[str]) -> str:
|
||||
value = match.group(0)
|
||||
try:
|
||||
ipaddress.IPv6Address(value)
|
||||
except ipaddress.AddressValueError:
|
||||
return value
|
||||
return REDACTION
|
||||
|
||||
|
||||
def _redact_fqdn(match: re.Match[str], allowed_hosts: set[str]) -> str:
|
||||
host = match.group(0)
|
||||
return host if host.lower().rstrip(".") in allowed_hosts else REDACTION
|
||||
|
||||
|
||||
def sanitize_text(text: str, *, allowed_hosts: Iterable[str]) -> str:
|
||||
hosts = _normalized_hosts(allowed_hosts)
|
||||
cleaned = _URL_RE.sub(lambda match: _redact_url(match, hosts), text)
|
||||
cleaned = _CREDENTIAL_RE.sub(REDACTION, cleaned)
|
||||
cleaned = _EMAIL_RE.sub(REDACTION, cleaned)
|
||||
cleaned = _PATH_RE.sub(REDACTION, cleaned)
|
||||
cleaned = _IPV4_RE.sub(_redact_ipv4, cleaned)
|
||||
cleaned = _IPV6_CANDIDATE_RE.sub(_redact_ipv6, cleaned)
|
||||
cleaned = _PHONE_RE.sub(REDACTION, cleaned)
|
||||
cleaned = _PRIVATE_HOST_RE.sub(REDACTION, cleaned)
|
||||
cleaned = _FQDN_RE.sub(lambda match: _redact_fqdn(match, hosts), cleaned)
|
||||
return cleaned
|
||||
|
||||
|
||||
def assert_publishable(text: str, *, allowed_hosts: Iterable[str]) -> None:
|
||||
sanitized = sanitize_text(text, allowed_hosts=allowed_hosts)
|
||||
if sanitized != text:
|
||||
raise ValueError("content still contains private or unapproved data")
|
||||
@@ -0,0 +1,50 @@
|
||||
import unittest
|
||||
|
||||
from page_state import MarkerError, find_marker
|
||||
|
||||
|
||||
VALID = "anwarpy: status=pending | access=free | category=نماذج | queued_at=2026-10-09T15:12:00+03:00"
|
||||
|
||||
|
||||
class PageStateTests(unittest.TestCase):
|
||||
def test_finds_one_exact_marker_after_unrelated_blocks(self):
|
||||
result = find_marker([
|
||||
{"id": "a", "text": "ملاحظات"},
|
||||
{"id": "b", "text": VALID},
|
||||
{"id": "c", "text": "مصدر عام"},
|
||||
])
|
||||
self.assertEqual(result.block_id, "b")
|
||||
self.assertEqual(result.meta["status"], "pending")
|
||||
|
||||
def test_returns_none_when_anwarpy_is_never_mentioned(self):
|
||||
self.assertIsNone(find_marker([{"id": "a", "text": "لا توجد علامة"}]))
|
||||
|
||||
def test_reports_every_near_miss(self):
|
||||
for text in (
|
||||
VALID.replace("anwarpy:", "Anwarpy:"),
|
||||
"prefix " + VALID,
|
||||
"documentation mentions anwarpy but has no marker",
|
||||
VALID.replace("anwarpy:", "anwarpy ="),
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
with self.assertRaisesRegex(MarkerError, "near-miss"):
|
||||
find_marker([{"id": "bad", "text": text}])
|
||||
|
||||
def test_scans_all_blocks_and_rejects_ambiguity(self):
|
||||
with self.assertRaisesRegex(MarkerError, "multiple"):
|
||||
find_marker([
|
||||
{"id": "first", "text": VALID},
|
||||
{"id": "middle", "text": "unrelated"},
|
||||
{"id": "last", "text": VALID.replace("category=نماذج", "category=أدوات")},
|
||||
])
|
||||
|
||||
def test_exact_plus_near_miss_is_ambiguous(self):
|
||||
with self.assertRaisesRegex(MarkerError, "ambiguous"):
|
||||
find_marker([
|
||||
{"id": "good", "text": VALID},
|
||||
{"id": "bad", "text": "Anwarpy: pending"},
|
||||
])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,79 @@
|
||||
import unittest
|
||||
|
||||
from publication_meta import (
|
||||
clean,
|
||||
mark_blocked,
|
||||
mark_published,
|
||||
mark_publishing,
|
||||
parse_meta,
|
||||
same_meta,
|
||||
)
|
||||
|
||||
|
||||
PENDING = "anwarpy: status=pending | access=vip | category=أدوات | queued_at=2026-10-09T15:12:00+03:00"
|
||||
|
||||
|
||||
class PublicationMetaTests(unittest.TestCase):
|
||||
def test_parses_pending_and_normalizes_formatting_controls(self):
|
||||
raw = "\ufeffanwarpy:\u00a0status=pending | access=vip | category=أدوات | queued_at=2026-10-09T15:12:00+03:00\u200f"
|
||||
meta = parse_meta(raw)
|
||||
self.assertEqual(meta["status"], "pending")
|
||||
self.assertEqual(meta["category"], "أدوات")
|
||||
|
||||
def test_rejects_invalid_values_and_dates(self):
|
||||
for raw in (
|
||||
PENDING.replace("access=vip", "access=paid"),
|
||||
PENDING.replace("category=أدوات", "category=عام"),
|
||||
PENDING.replace("2026-10-09", "2026-13-45"),
|
||||
PENDING.replace("+03:00", ""),
|
||||
):
|
||||
with self.subTest(raw=raw):
|
||||
with self.assertRaises(ValueError):
|
||||
parse_meta(raw)
|
||||
|
||||
def test_rejects_invalid_state_shapes(self):
|
||||
with self.assertRaisesRegex(ValueError, "slug"):
|
||||
parse_meta(PENDING.replace("status=pending", "status=publishing"))
|
||||
with self.assertRaisesRegex(ValueError, "blocked_reason"):
|
||||
parse_meta(PENDING.replace("status=pending", "status=blocked"))
|
||||
with self.assertRaisesRegex(ValueError, "not allowed"):
|
||||
parse_meta(PENDING + " | url=https://anwarpy.ai/posts/example")
|
||||
|
||||
def test_transitions_reparse_raw_input(self):
|
||||
publishing = mark_publishing(PENDING, "arabic-ai-guide")
|
||||
self.assertEqual(parse_meta(publishing)["status"], "publishing")
|
||||
published = mark_published(
|
||||
publishing,
|
||||
"2026-10-09T12:20:11+00:00",
|
||||
"https://anwarpy.ai/posts/arabic-ai-guide",
|
||||
)
|
||||
self.assertEqual(parse_meta(published)["status"], "published")
|
||||
|
||||
def test_blocked_is_terminal_and_carries_a_bounded_reason(self):
|
||||
blocked = mark_blocked(PENDING, "cover")
|
||||
self.assertEqual(parse_meta(blocked)["blocked_reason"], "cover")
|
||||
with self.assertRaises(ValueError):
|
||||
mark_blocked(PENDING, "network")
|
||||
|
||||
def test_canonical_url_and_slug_are_exact(self):
|
||||
publishing = mark_publishing(PENDING, "arabic-ai-guide")
|
||||
for url in (
|
||||
"http://anwarpy.ai/posts/arabic-ai-guide",
|
||||
"https://www.anwarpy.ai/posts/arabic-ai-guide",
|
||||
"https://anwarpy.ai/posts/arabic-ai-guide/",
|
||||
"https://anwarpy.ai/posts/other",
|
||||
):
|
||||
with self.subTest(url=url):
|
||||
with self.assertRaises(ValueError):
|
||||
mark_published(publishing, "2026-10-09T15:20:11+03:00", url)
|
||||
with self.assertRaises(ValueError):
|
||||
mark_publishing(PENDING, "عنوان-عربي")
|
||||
|
||||
def test_readback_comparison_is_semantic_after_cleaning(self):
|
||||
formatted = "\u200fanwarpy: status=pending|access=vip|category=أدوات|queued_at=2026-10-09T15:12:00+03:00"
|
||||
self.assertTrue(same_meta(PENDING, formatted))
|
||||
self.assertEqual(clean("\ufeff أداة\u00a0"), "أداة")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import unittest
|
||||
|
||||
from sanitize import assert_publishable, sanitize_text
|
||||
|
||||
|
||||
class SanitizeTests(unittest.TestCase):
|
||||
def test_redacts_private_identifiers_and_credentials(self):
|
||||
raw = """Connect to 10.1.2.3 or fd00::12 as admin@example.test.
|
||||
SSH root@private.internal and use password=FakeSecret123.
|
||||
Read /Users/example/private/config and call https://hidden.internal/api.
|
||||
Phone +1 555 123 4567; token: abcdefghijklmnop.
|
||||
"""
|
||||
cleaned = sanitize_text(raw, allowed_hosts={"anwarpy.ai"})
|
||||
for secret in (
|
||||
"10.1.2.3",
|
||||
"fd00::12",
|
||||
"admin@example.test",
|
||||
"root@private.internal",
|
||||
"FakeSecret123",
|
||||
"/Users/example/private/config",
|
||||
"hidden.internal",
|
||||
"+1 555 123 4567",
|
||||
"abcdefghijklmnop",
|
||||
):
|
||||
self.assertNotIn(secret, cleaned)
|
||||
assert_publishable(cleaned, allowed_hosts={"anwarpy.ai"})
|
||||
|
||||
def test_preserves_approved_public_urls(self):
|
||||
text = "راجع https://anwarpy.ai/posts/example للمزيد."
|
||||
self.assertEqual(sanitize_text(text, allowed_hosts={"anwarpy.ai"}), text)
|
||||
assert_publishable(text, allowed_hosts={"anwarpy.ai"})
|
||||
|
||||
def test_redacts_unapproved_public_hosts(self):
|
||||
cleaned = sanitize_text("Source: https://unapproved.example/article", allowed_hosts={"anwarpy.ai"})
|
||||
self.assertNotIn("unapproved.example", cleaned)
|
||||
assert_publishable(cleaned, allowed_hosts={"anwarpy.ai"})
|
||||
|
||||
def test_assert_publishable_raises_without_using_assert_statement(self):
|
||||
with self.assertRaises(ValueError):
|
||||
assert_publishable("internal address 192.168.1.10", allowed_hosts={"anwarpy.ai"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user