From beae7c154a450c3c9488af0441cd7d2b15223fbb Mon Sep 17 00:00:00 2001 From: ChemaVX Date: Wed, 24 Jun 2026 08:01:44 +0000 Subject: [PATCH] Initial commit: SEO rule engine, auditor, validator, exceptions, mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Versioning ~/seo-tools as critical SEO infrastructure (was home-dir only). Canonical home of seo_rules.py — ResearchOwl will vendor a copy with a CI diff guard against this repo. No secrets; Ghost access is via the ghst-en wrapper. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 6 ++ README.md | 15 ++++ phase2_apply.py | 51 +++++++++++ phase2_desc.json | 11 +++ seo_audit.py | 216 ++++++++++++++++++++++++++++++++++++++++++++++ seo_exceptions.py | 60 +++++++++++++ seo_mirror.py | 136 +++++++++++++++++++++++++++++ seo_rules.py | 203 +++++++++++++++++++++++++++++++++++++++++++ seo_validate.py | 146 +++++++++++++++++++++++++++++++ 9 files changed, 844 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 phase2_apply.py create mode 100644 phase2_desc.json create mode 100755 seo_audit.py create mode 100644 seo_exceptions.py create mode 100644 seo_mirror.py create mode 100644 seo_rules.py create mode 100644 seo_validate.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6bba25f --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +*.pyo +.env +*.token +*cache*.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..9a8ae8b --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# seo-tools + +SEO rule engine + tooling for the Ghost blogs (EN: theexclusionzone.com). + +- `seo_rules.py` — shared rule engine (pure functions, no I/O). Single source of truth + for thresholds + checks. Consumed by the auditor, the validator, and (vendored) by + ResearchOwl's blog pipeline. +- `seo_audit.py` — Tool A: site-wide auditor (reads all published posts via `ghst-en`). +- `seo_validate.py` — Tool C: single-post pre-publish validator (`seo-check` wraps this). +- `seo_exceptions.py` — documented, consciously-accepted waivers (auditor reporting filter). +- `seo_mirror.py` — OG/Twitter mirroring helper. +- `phase2_*.py/json` — one-off batch helpers (kept for provenance). + +No secrets live here. Ghost access is via the `ghst-en` wrapper (`~/.local/bin`), which +fetches a staff token on demand from the pod and never writes it to disk. diff --git a/phase2_apply.py b/phase2_apply.py new file mode 100644 index 0000000..edbf8c8 --- /dev/null +++ b/phase2_apply.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +""" +Phase 2 apply — trim 9 long meta_descriptions, keep og/twitter mirrored. + +Reads new descriptions from phase2_desc.json (slug -> new meta_description). +For each post sets meta_description = og_description = twitter_description to the +SAME new value via a minimal ghst --from-json patch (+ updated_at for Ghost's +collision check). Touches nothing else. READ-ONLY against meta_title/body/etc. +""" +import json, os, subprocess, sys, tempfile +from seo_audit import fetch_posts + +NEW = json.load(open(os.path.join(os.path.dirname(__file__), "phase2_desc.json"))) + +def apply_one(post, val): + body = {"posts": [{ + "meta_description": val, + "og_description": val, + "twitter_description": val, + "updated_at": post["updated_at"], + }]} + fd, jp = tempfile.mkstemp(suffix=".json"); os.close(fd) + fd, op = tempfile.mkstemp(suffix=".json"); os.close(fd) + try: + json.dump(body, open(jp, "w")) + cmd = f"ghst-en post update {post['id']} --from-json {jp} --json > {op}" + r = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + return r.stderr.strip() + return None + finally: + os.unlink(jp); os.unlink(op) + +def main(): + posts = {p["slug"]: p for p in fetch_posts()} + ok, fail = [], [] + for slug, val in NEW.items(): + p = posts.get(slug) + if not p: + fail.append((slug, "slug not found in live posts")); print(f" FAIL {slug}: not found"); continue + err = apply_one(p, val) + if err: + fail.append((slug, err)); print(f" FAIL {slug}: {err[:90]}") + else: + ok.append(slug); print(f" OK {slug} (meta/og/twitter desc = {len(val)} chars)") + print(f"\nApplied: {len(ok)} | Failed: {len(fail)}") + for s, e in fail: + print(f" FAILED {s}: {e}") + +if __name__ == "__main__": + main() diff --git a/phase2_desc.json b/phase2_desc.json new file mode 100644 index 0000000..a66a7a1 --- /dev/null +++ b/phase2_desc.json @@ -0,0 +1,11 @@ +{ + "rendlesham-forest-incident-1980": "Military witnesses, physical trace evidence, and official government files — the most thoroughly documented UFO case in British history.", + "david-grusch-capitol-2026-whistleblower-uap-disclosure": "Three years after his sworn testimony, Grusch returned to Capitol Hill demanding the declassification Congress still hasn't delivered.", + "malmstrom-1967-ufo-nuclear-missiles-incident": "In 1967, UFOs were seen over Malmstrom AFB as twenty nuclear missiles went offline at once. Boeing found no technical explanation.", + "spielberg-disclosure-day-2026-uap-film": "Spielberg's \"Disclosure Day\": how fifty years of research turned Hollywood's greatest UFO skeptic into a believer.", + "pursue-release-3-cia-confession-apollo-uap-files-2026": "The Pentagon's third UAP release: a CIA confession about spy planes, Apollo lunar anomalies, and 294 files spanning 82 years.", + "immaculate-constellation-secret-uap-program-congress": "A 2024 whistleblower document alleges a secret Pentagon UAP program. Nineteen months later, it's still neither confirmed nor denied.", + "aaro-ufo-investigation-eight-decades-us-government-research": "AARO found no alien technology after 80 years of investigation. What the Pentagon's UFO office discovered — and what it didn't.", + "gimbal-gofast-pentagon-analysis-uap-videos": "Five years after the Pentagon released GIMBAL and GOFAST, both remain unidentified. The data that would resolve them is still secret.", + "222-declassified-ufo-files-reveal-hide-unanswered-2026": "222 files, 82 years of records. What the Pentagon's largest UAP declassification reveals, what it hides, and what it leaves unanswered." +} diff --git a/seo_audit.py b/seo_audit.py new file mode 100755 index 0000000..3fc818c --- /dev/null +++ b/seo_audit.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Tool A — site-wide SEO auditor for theexclusionzone.com (EN, Ghost). + +READ-ONLY. Fetches every published post via the ghst-en wrapper (Ghost Admin API, +staff token pulled on-demand from the pod — nothing written to disk), runs the +shared rule engine (seo_rules.py), and prints: + 1. a summary of aggregate violations, + 2. a per-post table sorted worst-first, + 3. a ranked "fix this first" list. + +Usage: + python3 seo_audit.py # fetch live, full report + python3 seo_audit.py --json cache.json # audit a saved posts JSON instead of fetching + python3 seo_audit.py --save cache.json # fetch live AND save the raw JSON + +Changes NOTHING in Ghost. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile +from collections import Counter + +import seo_rules as R +import seo_exceptions as X + + +def partition(post, violations): + """Split one post's violations into (actionable, accepted, info). + + 'accepted' = an actionable-severity violation whose (rule, slug) is a + documented waiver in seo_exceptions — kept OUT of the to-fix count and shown + in its own section. The rule still ran; this is a reporting-side filter only. + """ + slug = post.get("slug") + actionable, accepted, info = [], [], [] + for v in violations: + if v.severity == R.INFO: + info.append(v) + elif X.accepted_reason(v.rule, slug): + accepted.append(v) + else: + actionable.append(v) + return actionable, accepted, info + + +def fetch_posts(): + """Pull all published posts (with html) via ghst-en. Returns list[dict]. + + We redirect ghst's stdout to a temp file rather than capturing the pipe: + ghst's output is ~750 KB and a captured PIPE truncates at the 64 KB pipe + buffer, whereas a file redirect gets the whole document. + """ + fd, path = tempfile.mkstemp(suffix=".json", prefix="seo_audit_") + os.close(fd) + try: + cmd = ("ghst-en post list --status published --limit 100 " + "--formats html,lexical --json > " + path) + r = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + sys.exit(f"ghst-en fetch failed:\n{r.stderr}") + data = json.load(open(path)) + return data["posts"] if isinstance(data, dict) and "posts" in data else data + finally: + os.unlink(path) + + +def load_posts(path): + data = json.load(open(path)) + return data["posts"] if isinstance(data, dict) and "posts" in data else data + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--json", help="audit a saved posts JSON instead of fetching live") + ap.add_argument("--save", help="save fetched raw posts JSON to this path") + args = ap.parse_args() + + if args.json: + posts = load_posts(args.json) + src = f"file {args.json}" + else: + posts = fetch_posts() + src = "live (ghst-en, Ghost Admin API)" + if args.save: + json.dump({"posts": posts}, open(args.save, "w"), indent=2) + + results = [] # (post, actionable, accepted, info, score) + for p in posts: + v = R.check_post(p) + actionable, accepted, info = partition(p, v) + # score ranks worst-first on ACTIONABLE severity only — accepted waivers + # and INFO don't push a post up the list. + results.append((p, actionable, accepted, info, R.score(actionable))) + + # worst-first: by actionable severity score, then actionable count, then slug + results.sort(key=lambda t: (-t[4], -len(t[1]), t[0]["slug"])) + + total_posts = len(posts) + # SUMMARY counts reflect ACTIONABLE violations only (waived items excluded). + posts_with_rule = Counter() + for _, actionable, _, _, _ in results: + for rule in {v.rule for v in actionable}: + posts_with_rule[rule] += 1 + + clean = sum(1 for _, actionable, _, _, _ in results if not actionable) + accepted_total = sum(len(acc) for _, _, acc, _, _ in results) + + # ---- header ---- + print("=" * 78) + print(f" SEO AUDIT — theexclusionzone.com source: {src}") + print(f" {total_posts} published posts | {clean} clean (no actionable issues)" + + (f" | {accepted_total} accepted exception(s) waived" if accepted_total else "")) + print(f" JSON-LD: handled globally by Edition-main theme — not flagged per post") + print("=" * 78) + + # ---- summary ---- + print("\nSUMMARY — violations by rule (most posts affected first):\n") + LABEL = { + "meta_description.missing": "meta_description MISSING", + "meta_description.too_long": f"meta_description > {R.META_DESC_MAX}", + "meta_title.missing": "meta_title missing (falls back to title)", + "meta_title.too_long": f"meta_title > {R.META_TITLE_MAX}", + "custom_excerpt.too_long": f"custom_excerpt > {R.CUSTOM_EXCERPT_MAX}", + "og_title.empty": "og_title empty", + "og_description.empty": "og_description empty", + "twitter_title.empty": "twitter_title empty", + "twitter_description.empty": "twitter_description empty", + "feature_image.missing": "feature_image missing", + "feature_image_alt.missing": "feature_image_alt missing", + "internal_links.too_few": f"internal links < {R.MIN_INTERNAL_LINKS}", + "title_eq_meta_title": "meta_title == title (info)", + } + # severity per rule (look one up from any occurrence, incl. waived ones) + sev_of = {} + for _, actionable, accepted, info, _ in results: + for v in actionable + accepted + info: + sev_of[v.rule] = v.severity + for rule, _ in sorted(posts_with_rule.items(), + key=lambda kv: (-sev_of.get(kv[0], 0), -kv[1])): + sev = R.SEV_NAME[sev_of.get(rule, 0)] + print(f" [{sev:4}] {posts_with_rule[rule]:2}/{total_posts} posts " + f"{LABEL.get(rule, rule)}") + + # ---- per-post table ---- + print("\n" + "-" * 78) + print("PER-POST (worst first) — score = summed severity (HIGH 3 / MED 2 / LOW 1)\n") + print(f"{'score':>5} {'slug':45} issues") + print(f"{'-'*5} {'-'*45} {'-'*20}") + for p, actionable, accepted, info, sc in results: + if not actionable: + note = "✓ clean" + (f" ({len(accepted)} accepted exception)" if accepted else "") + print(f"{sc:5} {p['slug'][:45]:45} {note}") + continue + tags = ", ".join(sorted({v.rule for v in actionable})) + print(f"{sc:5} {p['slug'][:45]:45} {tags}") + + # ---- detail per non-clean post ---- + print("\n" + "-" * 78) + print("DETAIL\n") + for p, actionable, accepted, info, sc in results: + if not actionable and not info: + continue + print(f"• {p['slug']} (score {sc})") + for v in sorted(actionable, key=lambda x: -x.severity): + print(f" [{R.SEV_NAME[v.severity]:4}] {v.message}" + + (f" → {v.fix}" if v.fix else "")) + for v in info: + print(f" [INFO] {v.message}") + print() + + # ---- accepted exceptions (won't fix — documented) ---- + if accepted_total: + print("-" * 78) + print("ACCEPTED EXCEPTIONS (won't fix — documented decisions, NOT in the count)\n") + for p, actionable, accepted, info, sc in sorted(results, key=lambda t: t[0]["slug"]): + for v in accepted: + reason = X.accepted_reason(v.rule, p["slug"]) + print(f"• {p['slug']}") + print(f" [{R.SEV_NAME[v.severity]:4}] {v.message}") + print(f" waived: {reason}") + print() + + # ---- ranked guidance ---- + print("-" * 78) + print("FIX FIRST (by impact / effort):\n") + md_missing = posts_with_rule.get("meta_description.missing", 0) + md_long = posts_with_rule.get("meta_description.too_long", 0) + links = posts_with_rule.get("internal_links.too_few", 0) + og = posts_with_rule.get("og_title.empty", 0) + alt = posts_with_rule.get("feature_image_alt.missing", 0) + rank = [] + if md_missing: + rank.append(f"meta_description MISSING on {md_missing} post(s) — HIGH impact, " + f"controls the Google snippet. Write unique 140–145-char descriptions.") + if md_long: + rank.append(f"meta_description > {R.META_DESC_MAX} on {md_long} post(s) — trim so the " + f"snippet isn't truncated (real ranking/CTR impact).") + if links: + rank.append(f"<{R.MIN_INTERNAL_LINKS} internal links on {links} post(s) — add 2–3 contextual " + f"links to related articles (helps crawl + topical authority).") + if og: + rank.append(f"Empty OG/Twitter fields on {og} post(s) — SAFEST AUTO-FIX: mirror " + f"meta_title/description into og_* and twitter_*. Highest count, lowest risk.") + if alt: + rank.append(f"Missing feature_image_alt on {alt} post(s) — quick accessibility + image-SEO win.") + for i, line in enumerate(rank, 1): + print(f" {i}. {line}") + print() + + +if __name__ == "__main__": + main() diff --git a/seo_exceptions.py b/seo_exceptions.py new file mode 100644 index 0000000..5f94575 --- /dev/null +++ b/seo_exceptions.py @@ -0,0 +1,60 @@ +""" +Accepted SEO exceptions — consciously-waived findings for SPECIFIC posts. + +This is an A-side (auditor) REPORTING filter, NOT a rule change. The shared rule +engine (seo_rules.py) still CHECKS every rule on every post; the auditor simply +moves a matching violation into a separate "Accepted exceptions (won't fix — +documented)" section so the headline "issues to fix" count reflects only +genuinely-actionable items. Each waiver carries its REASON (no silent skips). + +Structure: EXCEPTIONS[rule_family][slug] = "documented reason" + +Scope: + - A waiver applies ONLY to the named rule family for the named slug. Any OTHER + rule the same post violates still surfaces as a real issue. + - A waiver is matched against a Violation whose `rule` equals the family key or + starts with "." — so "internal_links" covers "internal_links.too_few". + - If a post later stops violating the rule (e.g. a link is added and it clears + the threshold), there's simply no violation to waive — the waiver is moot. + +Tool C (seo_validate.py, pre-publish validator) deliberately does NOT consult +this registry: exceptions are per-EXISTING-post decisions; a brand-new draft is +held to the full standard. +""" + +EXCEPTIONS = { + # internal_links < 2 — these posts are intentionally left as-is because there + # is no honest 2nd internal-link target in their prose; forcing one would hurt + # quality. (Decided 2026-06-23.) + "internal_links": { + "roswell-1947-ufo-incident": + "1 link (Spielberg); no 2nd entity in prose maps to an article — " + "forcing a link would hurt quality.", + "manises-1979-spain-ufo-incident": + "1 link (Canary Islands); no 2nd entity in prose maps to an article.", + "betty-barney-hill-1961-abduction": + "1 link (alien abduction → travis-walton, thematic); no 2nd honest target.", + "malmstrom-1967-ufo-nuclear-missiles-incident": + "1 link (Grusch); the only entity named in the prose.", + "rendlesham-forest-incident-1980": + "0 links; genuinely isolated — no entity in prose maps to one of our articles.", + "phoenix-lights-1997": + "0 links; only thematic candidates are stretches — left as-is.", + "canary-islands-1976-ufo-sighting": + "1 link (reciprocal → manises); no honest 2nd entity in prose — " + "Spanish case, self-contained.", + }, +} + + +def accepted_reason(rule, slug): + """Return the documented waiver reason if (rule, slug) is an accepted + exception, else None. + + Matches a rule-family key to a Violation whose `rule` is the key itself or + "." (e.g. family "internal_links" matches "internal_links.too_few"). + """ + for family, slugs in EXCEPTIONS.items(): + if (rule == family or rule.startswith(family + ".")) and slug in slugs: + return slugs[slug] + return None diff --git a/seo_mirror.py b/seo_mirror.py new file mode 100644 index 0000000..f426113 --- /dev/null +++ b/seo_mirror.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Tool A — OG/Twitter auto-mirror write path (theexclusionzone.com, Ghost). + +For each published post, mirror into EMPTY social fields only: + og_title <- meta_title (only if og_title empty) + twitter_title <- meta_title (only if twitter_title empty) + og_description <- meta_description (only if og_description empty) + twitter_description<- meta_description (only if twitter_description empty) + +Never overwrites a non-empty field. Touches ONLY those 4 fields (+ updated_at, +required by Ghost's optimistic-concurrency check). Idempotent: re-running only +fills what is still empty. + +Modes: + python3 seo_mirror.py --list # fetch live, show posts needing mirror + python3 seo_mirror.py --apply # apply to ONE post + python3 seo_mirror.py --apply-all [--skip a,b] # apply to all needing, minus skips + +Reuses fetch + emptiness logic from the audit engine. READ-ONLY in --list. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile + +import seo_rules as R +from seo_audit import fetch_posts + +SOCIAL = { + "og_title": "meta_title", + "twitter_title": "meta_title", + "og_description": "meta_description", + "twitter_description": "meta_description", +} + + +def build_patch(post): + """Return {field: source_value} for each EMPTY social field with a non-empty source.""" + patch = {} + for field, src in SOCIAL.items(): + if R._empty(post.get(field)): + srcval = post.get(src) + if not R._empty(srcval): + patch[field] = srcval + return patch + + +def needing(posts): + out = [] + for p in posts: + patch = build_patch(p) + if patch: + out.append((p, patch)) + return out + + +def apply_one(post, patch): + """PATCH one post via ghst-en with a minimal --from-json body. Returns ghst JSON.""" + body = {"posts": [dict(patch, updated_at=post["updated_at"])]} + fd, jpath = tempfile.mkstemp(suffix=".json", prefix="mirror_patch_") + os.close(fd) + fd, opath = tempfile.mkstemp(suffix=".json", prefix="mirror_out_") + os.close(fd) + try: + json.dump(body, open(jpath, "w")) + cmd = f"ghst-en post update {post['id']} --from-json {jpath} --json > {opath}" + r = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + return {"_error": r.stderr.strip()} + return json.load(open(opath)) + finally: + os.unlink(jpath) + os.unlink(opath) + + +def fmt_patch(patch): + return ", ".join(f"{k}←{('meta_title' if 'title' in k else 'meta_description')}" for k in patch) + + +def main(): + ap = argparse.ArgumentParser() + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--list", action="store_true") + g.add_argument("--apply", metavar="SLUG") + g.add_argument("--apply-all", action="store_true") + ap.add_argument("--skip", default="", help="comma-separated slugs to skip") + args = ap.parse_args() + + posts = fetch_posts() + todo = needing(posts) + skip = {s.strip() for s in args.skip.split(",") if s.strip()} + + if args.list: + print(f"{len(todo)} post(s) need the OG/Twitter mirror " + f"(of {len(posts)} published):\n") + for p, patch in sorted(todo, key=lambda t: t[0]["slug"]): + print(f" {p['slug'][:50]:50} fills: {', '.join(patch.keys())}") + return + + if args.apply: + match = [(p, pt) for p, pt in todo if p["slug"] == args.apply] + if not match: + sys.exit(f"'{args.apply}' not found among posts needing mirror " + f"(maybe already mirrored, or empty meta source).") + p, patch = match[0] + print(f"Applying to {p['slug']} ({p['id']}): {fmt_patch(patch)}") + res = apply_one(p, patch) + if "_error" in res: + sys.exit(f"FAILED: {res['_error']}") + print(" ghst returned OK.") + return + + # --apply-all + applied, failed = [], [] + for p, patch in sorted(todo, key=lambda t: t[0]["slug"]): + if p["slug"] in skip: + print(f" SKIP {p['slug']}") + continue + res = apply_one(p, patch) + if "_error" in res: + failed.append((p["slug"], res["_error"])) + print(f" FAIL {p['slug']}: {res['_error'][:80]}") + else: + applied.append(p["slug"]) + print(f" OK {p['slug']} ({', '.join(patch.keys())})") + print(f"\nApplied: {len(applied)} | Failed: {len(failed)} | Skipped: {len(skip)}") + if failed: + for s, e in failed: + print(f" FAILED {s}: {e}") + + +if __name__ == "__main__": + main() diff --git a/seo_rules.py b/seo_rules.py new file mode 100644 index 0000000..08d122a --- /dev/null +++ b/seo_rules.py @@ -0,0 +1,203 @@ +""" +Reusable SEO rule engine for The Exclusion Zone (EN) — theexclusionzone.com (Ghost). + +Pure functions, NO I/O. Feed it a Ghost Admin API post dict (with `html` format +included) and it returns a list of Violation(rule, severity, message, fix). + +Shared by: + - seo_audit.py — Tool A, site-wide auditor (this engine, run over all posts) + - (future) seo_validate.py — Tool C, pre-publish validator (same engine, one draft) + +Design note: every check is an independent function registered in RULES. To add a +rule, write a function (post) -> list[Violation] and append it to RULES. The auditor +and the validator both just call check_post(); they never re-implement a check. +""" +import re +from collections import namedtuple + +SITE_HOST = "theexclusionzone.com" + +# ---- thresholds (single source of truth, reused by validator) ------------- +META_TITLE_MAX = 60 +META_DESC_MAX = 145 +CUSTOM_EXCERPT_MAX = 300 +MIN_INTERNAL_LINKS = 2 + +# ---- severity weights (used to rank "worst first") ------------------------ +HIGH, MED, LOW, INFO = 3, 2, 1, 0 +SEV_NAME = {HIGH: "HIGH", MED: "MED", LOW: "LOW", INFO: "INFO"} + +# The Edition-main theme injects BlogPosting JSON-LD globally in default.hbs +# ({{#is "post"}} ...