Repository navigation
Expand file tree
/
Copy pathseo_lint.py
More file actions
executable file
·1073 lines (967 loc) · 52.8 KB
/
Copy pathseo_lint.py
File metadata and controls
executable file
·1073 lines (967 loc) · 52.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""CI gate for xerj.org's technical SEO invariants.
Implements the technical-SEO invariants that can be checked against the static
tree alone: the canonical/Open Graph/structured-data set, sitemap coverage and
the extensionless URL shape. They follow the sitemaps.org protocol and Google
Search Central's crawling-and-indexing guidance; each rule below carries its own
number and its own explanation. Rules 15, 22 and 63 need the live site (a
sitemap `<loc>` has to return 200) and are not implemented here — they belong in
a post-deploy job, not the merge gate. They now exist, in
`scripts/seo/verify_live.py`; run it after a deploy, never in CI.
Rule 67 is the boundary case between the two. It cannot see production either,
but it does not need to: Cloudflare Pages derives its not-found behaviour from
the published tree, so "does this site return a real 404" is answerable from
the committed files alone — and until 2026-08-22 the answer was no.
Exit codes
0 no gate violations
1 at least one gate violation (or a warning, with --warnings-as-errors)
python3 scripts/seo/seo_lint.py
python3 scripts/seo/seo_lint.py --warnings-as-errors
Dependencies: Python 3 stdlib only, plus `xmllint` if it happens to be on
PATH (rule 12 is skipped with a note when it is not). The HTML parsing is
`scripts/seo/audit.py`'s `PageParser` — one parser for the whole SEO
toolchain, so a parsing bug cannot make the auditor and the gate disagree.
"""
from __future__ import annotations
import argparse
import collections
import html
import json
import pathlib
import re
import shutil
import struct
import subprocess
import sys
import urllib.parse
import xml.etree.ElementTree as ET
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import audit as audit_mod # noqa: E402 (reuse the auditor's HTML parser)
import gen_sitemap # noqa: E402 (the sitemap is a deploy artifact, not a file)
import pagedata # noqa: E402
import urlmap # noqa: E402
SITEMAP_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
#: The exact `script-src` this site is allowed to ship. JSON-LD is a data
#: block (WHATWG HTML "prepare the script element" returns at step 13, before
#: the CSP inline check at step 21), so structured data never needs a CSP
#: change. Rule 55 exists to stop someone "fixing" that non-problem by
#: widening the policy. The `'unsafe-inline'` here predates this work and is
#: something to remove later, not to build on.
#:
#: ``https://*.googletagmanager.com`` is the GA4 gtag.js host emitted by
#: ``seo_head.ga4_snippet()``. It is a real script, not a JSON-LD workaround.
#: The inline config block uses the ``'unsafe-inline'`` token already listed.
ALLOWED_SCRIPT_SRC = frozenset({
"'self'", "blob:", "'unsafe-inline'", "https://static.cloudflareinsights.com",
"https://*.googletagmanager.com",
})
#: Internal links that are known-broken and consciously left that way.
#: Every entry needs a reason and an owner decision, not just a path.
KNOWN_BROKEN: dict[str, str] = {
"/resources/xerj-sales-kit.zip":
"the ZIP has never existed; the page promises '12 PDFs, ~2.4 MB' but the "
"12 PDFs on disk total 4.4 MB, so building it would contradict visible "
"copy this pass is not allowed to edit. Needs a human: ship a real kit "
"and correct the size, or repoint the CTA at the individual briefs.",
}
#: Rule 64 — file types under `landing/` that are not text and are therefore
#: not scanned for stale `.html` URLs. Everything else is scanned, including
#: files with no extension at all (`landing/get`).
_BINARY_SUFFIXES = frozenset({
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz",
".mp4", ".webm", ".woff", ".woff2", ".wasm",
})
#: Rule 64 allowlist — the only literals under `landing/` that may still spell
#: a `.html` path. Keys are landing-relative file paths; each value maps the
#: exact allowed literal to why it must stay. The list is deliberately keyed
#: per-file and per-literal: a new `.html` URL anywhere, including in these two
#: files, is still a gate violation.
#:
#: Both entries are Cloudflare Pages *configuration*, not published links. A
#: `_headers` rule head and a `_redirects` comment are matched against, or
#: describe, the on-disk asset path — rewriting them extensionless would either
#: silently stop applying (`_headers`) or make the comment describe a file that
#: is not what ships.
HTML_PATH_ALLOWLIST: dict[str, dict[str, str]] = {
"_headers": {
"/index.html":
"`_headers` rule head. The root payload is served at both `/` and "
"`/index.html`; the file has a rule for each, and the asset path "
"has to be spelled out for the second one to match.",
"/playground/*.html":
"`_headers` rule head — a Cloudflare glob matched against the "
"request path for the dashboards SPA's HTML assets, not a link.",
},
"_redirects": {
"/product.html":
"comment only: the file-layout header block names the files on "
"disk, not the URLs they are served at.",
"/brand.html":
"comment only: same file-layout header block.",
"/playground.html":
"comment only: the header block, plus the note recording why "
"pointing an alias at `/playground.html` cost two hops. Rewriting "
"the note extensionless would erase what it documents.",
},
}
#: Rule 64 — a `.html` path published as an absolute URL (`xerj.org/x.html`,
#: with or without a scheme) or as a root-relative path (`/x.html`). The
#: root-relative pattern's lookbehind keeps it from re-matching the path half
#: of a URL that the first pattern already covers, and from firing inside an
#: off-origin URL such as `https://github.com/o/r/blob/main/x.html`.
_PATH_CHARS = r"[A-Za-z0-9._~!$&'()*+,;=:@%/*-]"
STALE_URL_RE = re.compile(rf"xerj\.org(?P<path>/{_PATH_CHARS}*\.html)")
STALE_PATH_RE = re.compile(rf"(?<![A-Za-z0-9._~%:/-])(?P<path>/{_PATH_CHARS}*\.html)")
#: Rule 58/59 (every <img> has width/height/alt) is `[gate]` in the research
#: doc but is reported here as a warning: `demo/index.html` ships 12 unsized
#: screenshots plus one `<img src="">`, and adding intrinsic dimensions to a
#: page with its own layout CSS is a rendering change, not a metadata change.
#: Flip this to False once those images carry intrinsic dimensions; the rule
#: itself is already written and will start gating with no other edit.
IMG_RULES_ARE_WARNINGS = True
TITLE_MAX = 60
DESC_MIN, DESC_MAX = 110, 165
MAX_DEPTH = 3
GENERIC_ANCHORS = {"click here", "here", "read more", "this", "link", "more"}
#: Rule 65 — a GitHub ref that cannot move. ``main``/``master`` are branches
#: too, but they are the repository's permanent trunk and survive a merge; a
#: tag and a commit SHA are immutable. Anything else — ``feat/agentic-seo``,
#: ``fix/x``, a personal branch — is deleted after merge and the link 404s.
#: ``corpus-hub`` is the one further exception: a long-lived registry branch
#: the corpus-hub launch (2026-10-01) points consumers and contributors at.
#: It is never merged-and-deleted — moving it IS the release mechanism — so
#: pinning to it is the intended behaviour, like a moving "latest" channel.
_PERMANENT_GITHUB_REF_RE = re.compile(
r"^(?:main|master|HEAD|corpus-hub"
r"|[0-9a-f]{7,40}"
r"|v?\d+(?:\.\d+)*(?:[-.][0-9A-Za-z.]+)*)$")
#: Rule 65 — GitHub URL shapes that carry a ref as their next path segment.
#: ``feat/agentic-seo`` captures as ``feat``, which is not a permanent ref, so
#: a branch-pinned URL trips the rule whether or not the branch name has a
#: slash in it.
_GITHUB_REF_URL_RE = re.compile(
r"https?://(?:"
r"github\.com/[\w.-]+/[\w.-]+/(?:blob|tree|raw|blame|edit|commits)/"
r"|raw\.githubusercontent\.com/[\w.-]+/[\w.-]+/"
r")(?P<ref>[^/\s\"'`)<>\]]+)")
#: Rule 66 — text inside an ``<em>``/``<strong>`` that is really a chopped-up
#: identifier. ``ignored_files=2, ignored_dirs=3`` used to render as
#: ``ignored<em>files=2, ignored</em>dirs=3`` because the generator applied
#: intraword ``_`` emphasis. The generator no longer does; this rule is the
#: backstop, and it keys on the giveaway that no prose italic has: a word
#: character welded to the tag on either side.
_INTRAWORD_EMPHASIS_RE = re.compile(
r"(?:\w<(?:em|strong)>)|(?:</(?:em|strong)>\w)")
#: Rule 67 — the file whose presence decides whether this site can return an
#: HTTP 404 at all. Cloudflare Pages: "If your project does not include a
#: top-level ``404.html``, Pages assumes that you are deploying a single-page
#: application" and answers every unmatched path with the root ``index.html``
#: and a 200. The path is landing-relative because that is what Pages
#: publishes verbatim.
NOT_FOUND_PAGE = "404.html"
#: Rule 67 — a `_redirects` source pattern that matches paths nothing else
#: claimed. ``/*`` is the shape the SPA rewrite (``/* /index.html 200``)
#: uses; ``/:splat``-style placeholders at the root are the same thing spelled
#: differently. A rule that matches everything re-creates the soft 404 that
#: `404.html` exists to remove, whatever status code it carries: with 200 it
#: is the soft 404 itself, and with a 3xx it turns every wrong URL into a
#: redirect to a real page, which Google also treats as a soft 404.
_CATCH_ALL_SOURCE_RE = re.compile(r"^/(?:\*|:[A-Za-z_][\w-]*)$")
class Issue:
__slots__ = ("level", "rule", "path", "message")
def __init__(self, level: str, rule: str, path: str, message: str):
self.level, self.rule, self.path, self.message = level, rule, path, message
def __str__(self) -> str:
return f"[{self.level.upper():4}] {self.rule:<28} {self.path}: {self.message}"
class Lint:
def __init__(self, root: pathlib.Path, repo: pathlib.Path):
self.root = root
self.repo = repo
self.issues: list[Issue] = []
self.pages: dict[str, audit_mod.Page] = {}
self.raw: dict[str, str] = {}
self._parse()
# -- helpers ------------------------------------------------------------
def gate(self, rule: str, path: str, msg: str) -> None:
self.issues.append(Issue("gate", rule, path, msg))
def warn(self, rule: str, path: str, msg: str) -> None:
self.issues.append(Issue("warn", rule, path, msg))
def _parse(self) -> None:
for rel in urlmap.iter_pages(self.root):
f = self.root / rel
src = f.read_text(encoding="utf-8", errors="replace")
self.raw[rel] = src
page = audit_mod.Page(path=rel, abs_path=f)
parser = audit_mod.PageParser(page)
parser.feed(src)
parser.close()
self.pages[rel] = page
@staticmethod
def _meta(page: audit_mod.Page, key: str) -> str | None:
return page.metas.get(key)
@staticmethod
def _png_size(path: pathlib.Path) -> tuple[int, int] | None:
"""Width/height from the IHDR chunk, or None when not a PNG.
Used by the og:image card check — every og card this site ships is a
1200x630 PNG rendered by scripts/seo/mk_og_card.py, and a card of the
wrong size fails open-graph validation silently."""
try:
with open(path, "rb") as fh:
head = fh.read(24)
except OSError:
return None
if len(head) < 24 or head[:8] != b"\x89PNG\r\n\x1a\n":
return None
return struct.unpack(">II", head[16:24])
# ── sitemap ↔ filesystem (rules 1-
10000
14) ───────────────────────────────────
def check_sitemap(self) -> None:
sm = self.root / "sitemap.xml"
p = "sitemap.xml"
if sm.exists():
data = sm.read_bytes()
else:
# sitemap.xml is a deploy artifact (deploy-pages.yml writes it into
# landing/ before the upload); it is not committed. Lint the bytes
# that deploy would ship instead of gating on the file's absence.
data = gen_sitemap.sitemap_for(self.root).encode("utf-8")
if data.startswith(b"\xef\xbb\xbf"):
self.gate("sitemap.bom", p, "file starts with a UTF-8 BOM (rule 13)")
try:
text = data.decode("utf-8")
except UnicodeDecodeError as e:
self.gate("sitemap.encoding", p, f"not valid UTF-8: {e} (rule 13)")
return
if len(data) > 50_000_000:
self.gate("sitemap.size", p, f"{len(data)} bytes exceeds 50 MB (rule 10)")
try:
tree = ET.fromstring(text)
except ET.ParseError as e:
self.gate("sitemap.malformed", p, f"XML does not parse: {e}")
return
if tree.tag != f"{SITEMAP_NS}urlset":
self.gate("sitemap.namespace", p,
f"root is {tree.tag}, expected {SITEMAP_NS}urlset (rule 11)")
if "<priority>" in text or "<changefreq>" in text:
self.gate("sitemap.dead_hints", p,
"<priority>/<changefreq> present — both confirmed ignored "
"by Google and Bing (rule 7)")
if re.search(r"<loc>[^<]*&(?!amp;|lt;|gt;|quot;|apos;|#)", text):
self.gate("sitemap.unescaped", p,
"unescaped '&' in a <loc> — makes the file malformed (rule 13)")
locs: list[str] = []
today = _today()
for url in tree.findall(f"{SITEMAP_NS}url"):
loc_el = url.find(f"{SITEMAP_NS}loc")
loc = (loc_el.text or "").strip() if loc_el is not None else ""
locs.append(loc)
if urllib.parse.urlsplit(loc).path.lower().endswith(".md"):
self.gate("sitemap.markdown", p,
f"{loc} is a Markdown twin; twins must stay out of sitemap.xml")
continue
if not loc.startswith(urlmap.SITE_ORIGIN + "/") and loc != urlmap.SITE_ORIGIN + "/":
self.gate("sitemap.not_absolute", p,
f"{loc!r} is not an absolute https://xerj.org URL (rule 5)")
continue
loc_path = urllib.parse.urlsplit(loc).path.rstrip("/")
if loc.endswith(".html") or loc_path == "/index" or loc_path.endswith("/index"):
self.gate("sitemap.html_form", p,
f"{loc} uses a redirecting URL form (rule 6)")
rel = urlmap.rel_for_url(loc)
if rel is None or not (self.root / rel).is_file():
self.gate("sitemap.dead_url", p,
f"{loc} corresponds to no file on disk (rule 2)")
elif not rel.endswith(".html"):
self.gate("sitemap.non_html", p,
f"{loc} is not an HTML page (rule 14)")
lm_el = url.find(f"{SITEMAP_NS}lastmod")
if lm_el is not None:
lm = (lm_el.text or "").strip()
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T.*)?", lm):
self.gate("sitemap.lastmod_format", p,
f"{loc}: lastmod {lm!r} is not W3C Datetime (rule 8)")
elif lm[:10] > today:
self.gate("sitemap.lastmod_future", p,
f"{loc}: lastmod {lm} is in the future (rule 9)")
if len(locs) > 50_000:
self.gate("sitemap.count", p, f"{len(locs)} URLs exceeds 50,000 (rule 10)")
dupes = [u for u, n in collections.Counter(locs).items() if n > 1]
for u in dupes:
self.gate("sitemap.duplicate", p, f"{u} appears more than once (rule 3)")
expected = {urlmap.deployed_url(rel) for rel in self.pages
if urlmap.in_sitemap(rel)}
for missing in sorted(expected - set(locs)):
self.gate("sitemap.missing_page", p,
f"{missing} is an indexable page but is not listed (rule 1)")
# rule 4 — the file must be generated, not hand-edited
dates = urlmap.DateSource(self.repo)
if text != gen_sitemap.build(self.root, dates):
self.gate("sitemap.stale", p,
"differs from `gen_sitemap.py --write` output — it was "
"hand-edited or is out of date (rule 4)")
# rule 12 — XSD validation, when xmllint is available
xsd = self.repo / "schemas" / "sitemap.xsd"
if shutil.which("xmllint") and xsd.exists():
# Validate the bytes under test, not the path: when the sitemap is
# generated in memory (it is a deploy artifact, not a committed
# file) there is no file for xmllint to open.
r = subprocess.run(["xmllint", "--noout", "--schema", str(xsd), "-"],
input=text, capture_output=True, text=True)
if r.returncode != 0:
self.gate("sitemap.xsd", p, f"xmllint: {r.stderr.strip()} (rule 12)")
# ── robots.txt (rules 38, 39) ───────────────────────────────────────────
def check_robots(self) -> None:
f = self.root / "robots.txt"
p = "robots.txt"
if not f.exists():
self.gate("robots.missing", p, "landing/robots.txt does not exist")
return
text = f.read_text(encoding="utf-8")
if f"Sitemap: {urlmap.SITE_ORIGIN}/sitemap.xml" not in text:
self.gate("robots.sitemap", p,
f"no `Sitemap: {urlmap.SITE_ORIGIN}/sitemap.xml` line (rule 39)")
disallows = [m.group(1).strip()
for m in re.finditer(r"^\s*Disallow:\s*(\S+)", text, re.M)]
for rel in urlmap.NOINDEX:
path = urlmap.deployed_path(rel)
for d in disallows:
if d and path.startswith(d):
self.gate("robots.noindex_blocked", p,
f"Disallow: {d} hides {path}, whose noindex would "
f"then never be seen (rule 38)")
# ── CSP (rule 55) ───────────────────────────────────────────────────────
def check_headers(self) -> None:
f = self.root / "_headers"
p = "_headers"
if not f.exists():
self.warn("headers.missing", p, "landing/_headers does not exist")
return
for m in re.finditer(r"script-src([^;]*)", f.read_text(encoding="utf-8")):
for src in m.group(1).split():
if src not in ALLOWED_SCRIPT_SRC:
self.gate("csp.script_src_widened", p,
f"script-src gained {src!r}. JSON-LD is a data "
f"block and never needs a CSP change (rule 55)")
# ── stale .html URLs in any published file (rule 64) ────────────────────
def check_published_urls(self) -> None:
"""No file Cloudflare publishes may spell a `.html` URL.
Rule 21 covers `<a href>` in HTML. It cannot see the rest of what
ships: `llms.txt` and `llms-full.txt` are the agent-facing entry points
to the site and are pure text, and an absolute `https://xerj.org/x.html`
is not an "internal href" at all, so it slipped past the site-wide
migration in two print-only footers. This rule reads every published
text file byte-for-byte, so there is nowhere left for a redirecting URL
to hide.
Overlap with rule 21 on root-relative hrefs inside HTML is deliberate:
one rule reasons about parsed links, this one about published bytes,
and a regression should trip both rather than neither.
"""
for f in sorted(self.root.rglob("*")):
if not f.is_file() or f.suffix.lower() in _BINARY_SUFFIXES:
continue
rel = f.relative_to(self.root).as_posix()
try:
text = f.read_text(encoding="utf-8")
except (UnicodeDecodeError, OSError):
continue # not a text file we can reason about
allowed = HTML_PATH_ALLOWLIST.get(rel, {})
for regex, kind in ((STALE_URL_RE, "absolute URL"),
(STALE_PATH_RE, "root-relative path")):
for m in regex.finditer(text):
path = m.group("path")
if path in allowed:
continue
line = text.count("\n", 0, m.start()) + 1
served = self._served_form(path)
self.gate("url.html_form", rel,
f"line {line}: {kind} {m.group(0)!r} ends in "
f"`.html` — Cloudflare 308s it to {served} "
f"(rule 64)")
@staticmethod
def _served_form(path: str) -> str:
"""The URL Cloudflare actually serves for a `.html` path, for the
message text. Falls back to the literal when the path is not a page
(a glob, say), so the report never invents a target."""
target = urlmap.rel_for_path(path)
if target is None or "*" in path:
return path
try:
return urlmap.deployed_path(target)
except ValueError:
return path
# ── per-page head, canonical, structured data ───────────────────────────
# ── branch-pinned GitHub URLs (rule 65) ─────────────────────────────────
def check_pinned_refs(self) -> None:
"""No published or authored file may cite a mutable GitHub branch.
Three writing agents emitted evidence sources as
``https://github.com/xerj-org/xerj/blob/feat/agentic-seo/<path>`` while
working around a build bug. That branch is deleted on merge, so every
one of those links dies the day the PR lands. A repo-relative path is
the correct form for in-repo evidence; when a URL really is wanted it
has to name a ref that survives — ``main``, a tag, or a commit SHA.
Both trees are scanned. ``content/`` is where the mistake is authored
and ``landing/`` is where it ships, and a regression should be caught
at the source rather than only in the generated output.
"""
for label, base in (("content", self.repo / "content"),
("landing", self.root)):
if not base.is_dir():
continue
for f in sorted(base.rglob("*")):
if not f.is_file() or f.suffix.lower() in _BINARY_SUFFIXES:
continue
try:
text = f.read_text(encoding="utf-8")
except (UnicodeDecodeError, OSError):
continue
rel = f.relative_to(base).as_posix()
if label == "content":
rel = f"../content/{rel}"
for m in _GITHUB_REF_URL_RE.finditer(text):
ref = m.group("ref")
if _PERMANENT_GITHUB_REF_RE.fullmatch(ref):
continue
line = text.count("\n", 0, m.start()) + 1
self.gate("url.branch_pinned", rel,
f"line {line}: {m.group(0)!r} pins a mutable "
f"GitHub ref {ref!r}. The branch is deleted "
f"after merge and the link 404s — use a "
f"repo-relative path, or blob/main, a tag or a "
f"commit SHA (rule 65)")
# ── identifiers italicised by intraword `_` (rule 66) ───────────────────
def check_intraword_emphasis(self) -> None:
"""No ``<em>``/``<strong>`` may be welded to a surrounding word.
Genuine prose italics always sit at a word boundary. An emphasis tag
with a word character immediately against it is the signature of an
identifier that was split by intraword ``_`` emphasis, which is a
rendering bug rather than a writing choice.
"""
for f in sorted(self.root.rglob("*")):
if not f.is_file() or f.suffix.lower() not in (".html", ".md"):
continue
try:
text = f.read_text(encoding="utf-8")
except (UnicodeDecodeError, OSError):
continue
rel = f.relative_to(self.root).as_posix()
for m in _INTRAWORD_EMPHASIS_RE.finditer(text):
line = text.count("\n", 0, m.start()) + 1
snippet = text[max(0, m.start() - 40):m.end() + 40].replace("\n", " ")
self.gate("emphasis.intraword", rel,
f"line {line}: {m.group(0)!r} — an emphasis tag is "
f"welded to a word character, which means an "
f"identifier was italicised by intraword `_` "
f"emphasis: {snippet!r} (rule 66)")
# ── the site can return a real 404 (rule 67) ────────────────────────────
def check_not_found(self) -> None:
"""`landing/404.html` must exist and nothing may swallow unknown paths.
Cloudflare Pages publishes `landing/` verbatim and decides its
not-found behaviour from the tree alone. With no top-level
`404.html` it treats the project as a single-page application and
answers **every** unmatched path with the root `index.html` and an
HTTP **200**. That is a soft 404: a typo'd URL, a dead link and a
URL an AI crawler guessed all return a plausible page instead of an
error, every one of them a byte-identical duplicate of the homepage,
and a removed page can never be signalled as removed. Verified on
the live site on 2026-08-22, before this file existed:
/ 200 21841 B ETag 93477f59…
/this-page-has-never-existed… 200 21841 B ETag 93477f59…
/answers/what-is-xerj 200 21841 B ETag 93477f59…
Three URLs, one of which has never existed, one body, one ETag.
The second half of the rule is the way the same failure comes back
after `404.html` is added: one catch-all line in `_redirects`.
`/* /index.html 200` restores the soft 404 exactly, and a catch-all
with a 3xx status is the redirect-shaped variant Google also files as
a soft 404. Named prefixes (`/docs/*`) are fine — they claim a real
section; only a source that matches every path is a violation.
"""
page = self.root / NOT_FOUND_PAGE
if not page.is_file():
self.gate("notfound.missing", NOT_FOUND_PAGE,
"the site has no top-level 404.html, so Cloudflare Pages "
"serves index.html with HTTP 200 for every unknown path — "
"a sitewide soft 404 (rule 67)")
redirects = self.root / "_redirects"
if not redirects.is_file():
return
for n, raw in enumerate(redirects.read_text(encoding="utf-8").splitlines(), 1):
line = raw.split("#", 1)[0].strip()
if not line:
continue
source = line.split()[0]
if _CATCH_ALL_SOURCE_RE.fullmatch(source):
self.gate("redirects.catch_all", "_redirects",
f"line {n}: {line!r} — a catch-all source matches every "
f"path nothing else claimed, so unknown URLs never reach "
f"404.html and the site soft-404s again (rule 67)")
def check_pages(self) -> None:
titles: dict[str, list[str]] = collections.defaultdict(list)
descs: dict[str, list[str]] = collections.defaultdict(list)
for rel, page in self.pages.items():
src = self.raw[rel]
url = urlmap.deployed_url(rel)
head = _head_of(src)
# -- canonical (16-19)
if len(page.canonicals) == 0:
self.gate("canonical.missing", rel, "no <link rel=canonical> (rule 16)")
elif len(page.canonicals) > 1:
self.gate("canonical.conflict", rel,
f"{len(page.canonicals)} canonicals (rules 16, 35)")
else:
href = page.canonicals[0][0].strip()
if f'rel="canonical" href="{href}"' not in head and href not in head:
self.gate("canonical.not_in_head", rel,
"canonical is outside <head>, so Google ignores it (rule 16)")
if href != url:
self.gate("canonical.mismatch", rel,
f"canonical {href} != deployed URL {url} (rule 17)")
if href.endswith(".html"):
self.gate("canonical.html", rel,
f"canonical {href} ends in .html and 308-redirects (rule 19)")
if "#" in href or "?" in href or not href.startswith("https://"):
self.gate("canonical.form", rel,
f"canonical {href} must be absolute https with no "
f"fragment or query (rule 18)")
# -- title (23, 24, 25, 35)
title = (page.title or "").strip()
if not title:
self.gate("title.missing", rel, "empty or absent <title> (rule 23)")
else:
titles[title].append(rel)
if len(title) > TITLE_MAX:
self.warn("title.long", rel,
f"<title> is {len(title)} chars (>{TITLE_MAX}) (rule 25)")
if page.titles_seen > 1:
self.gate("title.multiple", rel,
f"{page.titles_seen} <title> elements (rule 35)")
# -- description (26, 27, 28, 35)
desc = (page.description or "").strip()
if not desc:
self.gate("desc.missing", rel, "no meta description (rule 26)")
else:
descs[desc].append(rel)
if not (DESC_MIN <= len(desc) <= DESC_MAX):
self.warn("desc.size", rel,
f"description is {len(desc)} chars, target "
f"{DESC_MIN}-{DESC_MAX} (rule 28)")
if len(re.findall(r'<meta\b[^>]*name\s*=\s*["\']description["\']',
head, re.I)) > 1:
self.gate("desc.multiple", rel, "more than one description (rule 35)")
# -- keywords (29)
if re.search(r'<meta\b[^>]*name\s*=\s*["\']keywords["\']', src, re.I):
self.gate("meta.keywords", rel,
"<meta name=keywords> — Google: 'no effect on indexing "
"and ranking at all' (rule 29)")
# -- charset / viewport (34)
if not re.search(r"<meta\b[^>]*charset", head, re.I):
self.gate("charset.missing", rel, "no <meta charset> (rule 34)")
if not page.viewport:
self.gate("viewport.missing", rel, "no viewport meta (rule 34)")
# -- Day/Night toggle (#1973): one shared script remembers the choice.
if 'class="theme"' in src:
if '<script src="/theme.js"></script>' not in head:
self.gate("theme.script_missing", rel,
"page has the DAY/NIGHT toggle but <head> does not load "
"/theme.js, so the choice is lost on the next page (#1973)")
if "root.setAttribute('data-theme'" in src:
self.gate("theme.inline_toggle", rel,
"inline DAY/NIGHT handler duplicates /theme.js and never "
"saves the choice; remove it (#1973)")
# -- Open Graph + Twitter (30, 31, 32)
for k in ("og:title", "og:type", "og:url", "og:image", "og:description"):
if not self._meta(page, k):
self.gate("og.missing", rel, f"no {k} (rule 30)")
og_url = self._meta(page, "og:url")
if og_url and og_url != url:
self.gate("og.url_mismatch", rel,
f"og:url {og_url} != canonical {url} (rule 20)")
og_img = self._meta(page, "og:image")
if og_img:
if not og_img.startswith("https://"):
self.gate("og.image_relative", rel,
f"og:image {og_img} is not absolute https (rule 31)")
else:
asset = urllib.parse.urlsplit(og_img).path.lstrip("/")
if not (self.root / asset).is_file():
self.gate("og.image_missing", rel,
f"og:image {og_img} has no file on disk (rule 31)")
elif not asset.lower().endswith(".png"):
self.warn("og.image_not_png", rel,
f"og:image {og_img} is not a PNG — size "
f"not checked (rule 31)")
else:
size = self._png_size(self.root / asset)
if size is None:
self.gate("og.image_not_png", rel,
f"og:image {og_img} does not parse as a "
f"PNG (rule 31)")
elif size != (1200, 630):
self.gate("og.image_size", rel,
f"og:image {og_img} is {size[0]}x{size[1]}, "
f"not the 1200x630 card size (rule 31)")
card = self._meta(page, "twitter:card")
if card not in ("summary", "summary_large_image"):
self.gate("twitter.card", rel,
f"twitter:card is {card!r} (rule 32)")
# -- robots meta (33, 36, 37)
robots = (page.robots or "").lower()
if robots:
tokens = {t.strip() for t in robots.split(",")}
if tokens <= {"index", "follow"} and tokens:
self.gate("robots.noop", rel,
f"meta robots {robots!r} is the default — a no-op "
f"tag whose only effect is a chance to typo "
f"'noindex' (rule 33)")
declared_noindex = "noindex" in robots
if declared_noindex != urlmap.is_noindex(rel):
self.gate("robots.noindex_drift", rel,
f"page {'declares' if declared_noindex else 'does not declare'}"
f" noindex but urlmap.NOINDEX says otherwise")
if declared_noindex and urlmap.in_sitemap(rel):
self.gate("robots.noindex_in_sitemap", rel,
"noindex page is advertised in sitemap.xml (rule 36)")
# -- structured data (40-48)
self._check_jsonld(rel, page, url, title, desc)
# -- fonts (56, 57)
self._check_fonts(rel, src, head)
# -- images (58, 59)
for img in page.images:
if img.get("alt") is None:
self._img_issue("img.no_alt", rel,
f"line {img['line']}: <img> without alt (rule 59)")
if not img.get("width") or not img.get("height"):
self._img_issue("img.no_dims", rel,
f"line {img['line']}: <img> without width/height "
f"(rule 58)")
for t, rels in titles.items():
if len(rels) > 1:
self.gate("title.duplicate", rels[0],
f"<title> {t!r} is shared with {', '.join(rels[1:])} (rule 24)")
for d, rels in descs.items():
if len(rels) > 1:
self.gate("desc.duplicate", rels[0],
f"description shared with {', '.join(rels[1:])} (rule 27)")
def check_markdown_twins(self) -> None:
"""Check the HTML alternate contract without treating twins as pages."""
for rel, src in self.raw.items():
if not re.fullmatch(r"(?:answers|compare)/(?!index\.html$)[a-z0-9-]+\.html", rel):
continue
expected_href = "/" + rel[:-len(".html")] + ".md"
twin = self.root / expected_href.lstrip("/")
if not twin.is_file():
self.gate("markdown.missing", rel,
f"Markdown twin {expected_href} is missing")
tags = re.findall(r"<link\b[^>]*>", _head_of(src), re.I)
alternates = []
for tag in tags:
rel_match = re.search(r"\brel\s*=\s*[\"']([^\"']*)[\"']", tag, re.I)
type_match = re.search(r"\btype\s*=\s*[\"']([^\"']*)[\"']", tag, re.I)
href_match = re.search(r"\bhref\s*=\s*[\"']([^\"']*)[\"']", tag, re.I)
rels = (rel_match.group(1).lower().split() if rel_match else [])
if "alternate" in rels and type_match and type_match.group(1).lower() == "text/markdown":
alternates.append(href_match.group(1) if href_match else "")
if alternates != [expected_href]:
self.gate("markdown.alternate", rel,
f"expected one text/markdown alternate href {expected_href!r}; got {alternates!r}")
if re.search(r"<a\b[^>]*\bhref\s*=\s*[\"'](?![A-Za-z][A-Za-z0-9+.-]*:|//)[^\"']+\.md(?:[?#][^\"']*)?[\"']",
src, re.I):
self.gate("markdown.ordinary_link", rel,
"Markdown twins must be referenced with <link rel=alternate>, not <a>")
def _img_issue(self, rule: str, rel: str, msg: str) -> None:
(self.warn if IMG_RULES_ARE_WARNINGS else self.gate)(rule, rel, msg)
def _check_fonts(self, rel: str, src: str, head: str) -> None:
if "fonts.googleapis.com" not in src:
return
if not re.search(r'<link[^>]*rel=["\']preconnect["\'][^>]*'
r'fonts\.googleapis\.com', head, re.I):
self.gate("fonts.preconnect", rel,
"loads fonts.googleapis.com without a preconnect (rule 56)")
m = re.search(r'<link[^>]*fonts\.gstatic\.com[^>]*>', head, re.I)
if not m or "crossorigin" not in m.group(0).lower():
self.gate("fonts.preconnect_crossorigin", rel,
"fonts.gstatic.com preconnect missing or lacks the required "
"crossorigin attribute (rule 56)")
for u in re.findall(r'https://fonts\.googleapis\.com/css2\?[^"\']+', src):
if "display=swap" not in u:
self.gate("fonts.display_swap", rel,
"Google Fonts URL lacks display=swap (rule 57)")
def _visible_text(self, rel: str) -> str:
raw = (self.root / rel).read_text(encoding="utf-8")
raw = re.sub(r"(?is)<head\b.*?</head>", " ", raw)
raw = re.sub(r"(?is)<(script|style)\b.*?</\1>", " ", raw)
raw = re.sub(r"(?i)</?(?:a|b|code|em|i|kbd|span|strong)\b[^>]*>", "", raw)
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
return re.sub(r"\s+", " ", html.unescape(raw)).strip()
def _check_faq_visible(self, rel: str, line: int, node: dict) -> None:
"""Rule 46b: an opted-in FAQPage must mirror questions shown on the page."""
text = self._visible_text(rel)
for q in node.get("mainEntity") or []:
name = (q or {}).get("name", "")
answer = ((q or {}).get("acceptedAnswer") or {}).get("text", "")
if not name or name not in text:
self.gate("jsonld.faq_not_visible", rel,
f"line {line}: FAQPage question {name[:60]!r} is not "
f"visible on the page (rule 46b)")
elif not answer or answer[:80] not in text:
self.gate("jsonld.faq_not_visible", rel,
f"line {line}: FAQPage answer to {name[:60]!r} does not "
f"match the visible answer (rule 46b)")
def _check_jsonld(self, rel: str, page: audit_mod.Page, url: str,
title: str, desc: str) -> None:
if not page.jsonld:
self.gate("jsonld.missing", rel, "no application/ld+json block (rule 40)")
return
types: set[str] = set()
crumbs: list[dict] = []
articles: list[dict] = []
for raw, line in page.jsonld:
try:
obj = json.loads(raw)
except json.JSONDecodeError as e:
self.gate("jsonld.invalid", rel,
f"line {line}: block does not parse as JSON: {e} (rule 40)")
continue
for node in obj if isinstance(obj, list) else [obj]:
if not isinstance(node, dict):
self.gate("jsonld.not_object", rel,
f"line {line}: JSON-LD is not an object (rule 41)")
continue
if node.get("@context") != "https://schema.org":
self.gate("jsonld.context", rel,
f"line {line}: @context is "
f"{node.get('@context')!r} (rule 41)")
t = node.get("@type")
if not t:
self.gate("jsonld.no_type", rel,
f"line {line}: no @type (rule 41)")
continue
types.add(t)
if t in ("FAQPage", "HowTo"):
# Opt-in only (README trap 6): a page whose article source
# lists the type under `structured_data:` may carry it.
opted = pagedata.entry(rel).get("structured_data") or ()
kind = "faq" if t == "FAQPage" else "howto"
if kind not in opted:
self.gate("jsonld.deprecated_type", rel,
f"line {line}: {t} — FAQ rich results end 7 May 2026 "
f"and HowTo was removed outright (rule 46); a page "
f"may carry it only by listing `{kind}` under "
f"structured_data: in its content source")
elif t == "FAQPage":
self._check_faq_visible(rel, line, node)
if "aggregateRating" in node or "review" in node:
self.gate("jsonld.fabricated_rating", rel,
f"line {line}: aggregateRating/review without a real "
f"review corpus is a manual-action risk")
if t == "BreadcrumbList":
crumbs.append(node)
if t in ("Article", "TechArticle", "NewsArticle", "BlogPosting"):
articles.append(node)
# rule 42
if rel != "index.html" and not crumbs:
self.gate("jsonld.no_breadcrumb", rel,
"page below the root has no BreadcrumbList (rule 42)")
# rules 43, 44, 45
for bc in crumbs:
items = bc.get("itemListElement") or []
if not isinstance(items, list) or not items:
self.gate("breadcrumb.empty", rel, "BreadcrumbList has no items (rule 43)")
continue
for i, it in enumerate(items):
pos = it.get("position")
if pos != i + 1:
self.gate("breadcrumb.position", rel,
f"item {i}: position {pos!r}, expected {i + 1} (rule 43)")
if not it.get("name"):
self.gate("breadcrumb.name", rel, f"item {i} has no name (rule 44)")
item_url = it.get("item")
if i < len(items) - 1:
if not isinstance(item_url, str) or not item_url.startswith("https://"):
self.gate("breadcrumb.item", rel,
f"item {i} needs an absolute item URL (rule 44)")
continue
if item_url.endswith(".html"):
self.gate("breadcrumb.item_html", rel,
f"item {i}: {item_url} ends in .html (rule 45)")
target = urlmap.rel_for_url(item_url)
if target is None or not (self.root / target).is_file():
self.gate("breadcrumb.item_dead", rel,
f"item {i}: {item_url} is not a real page (rule 45)")
elif item_url is not None:
self.warn("breadcrumb.last_item", rel,
"the final ListItem should omit `item` so Google "
"uses the containing page URL (rule 44)")
# rules 47, 48
kind = pagedata.entry(rel)["kind"]
if kind in pagedata.ARTICLE_KINDS:
if not articles:
self.gate("jsonld.no_article", rel,
f"{kind} page carries no Article/TechArticle (rule 47)")
for a in articles:
for field in ("headline", "description", "datePublished",
"dateModified", "mainEntityOfPage"):
if not a.get(field):
self.gate("article.field", rel,
f"Article is missing {field} (rule 47)")
meop = a.get("mainEntityOfPage")
meop_url = meop.get("@id") if isinstance(meop, dict) else meop
if meop_url != url:
self.gate("article.main_entity", rel,
f"mainEntityOfPage {meop_url!r} != canonical {url} "
f"(rule 48)")
# ── internal link graph (rules 21, 49, 50, 51, 52, 53, 54) ──────────────
def check_links(self) -> None:
graph: dict[str, set[str]] = {rel: set() for rel in self.pages}
anchor_text: dict[str, list[tuple[str, str]]] = collections.defaultdict(list)
for rel, page in self.pages.items():
src_dir = pathlib.PurePosixPath(rel).parent
for href, line, _rel_attr in page.links:
href = html.unescape(href.strip())
if not urlmap.is_internal_href(href):
continue
path = href.split("#", 1)[0].split("?", 1)[0]
if not path:
continue
if path.endswith(".html"):
self.gate("link.html_form", rel,
f"line {line}: href {href!r} costs a 308 — link the "
f"served URL instead (rule 21)")
target = self._resolve(path, src_dir)
if target is None:
if path == "/sitemap.xml":
# Written by deploy-pages.yml just before the upload, so
# it is never a file in the tree; the artifact has it.
continue
if path in KNOWN_BROKEN:
self.warn("link.known_broken", rel,
f"line {line}: {path} — {KNOWN_BROKEN[path]}")
else:
self.gate("link.broken", rel,
f"line {line}: {href!r} resolves to nothing (rule 51)")
continue
if target.endswith(".html") and target in self.pages:
graph[rel].add(target)
for tag in re.finditer(r"<a\b[^>]*>(.*?)</a>", self.raw[rel], re.I | re.S):
text = " ".join(re.sub(r"<[^>]+>", " ", tag.group(1)).split()).lower()
if text in GENERIC_ANCHORS:
anchor_text[rel].append((text, tag.group(0)[:60]))
# rules 49, 50 — BFS from the homepage
depth = {"index.html": 0}
queue = collections.deque(["index.html"])
while queue:
cur = queue.popleft()
for nxt in sorted(graph.get(cur, ())):
if nxt not in depth:
depth[nxt] = depth[cur] + 1
queue.append(nxt)
for rel in sorted(self.pages):
if not urlmap.in_sitemap(rel) and rel not in depth:
continue # deliberately unlisted pages need no crawl path
if rel not in depth:
self.gate("link.orphan", rel,
"unreachable from index.html by following <a href> (rule 49)")
elif depth[rel] > MAX_DEPTH:
self.gate("link.deep", rel,
f"{depth[rel]} clicks from the homepage (max {MAX_DEPTH}) "
f"(rule 50)")
# rule 54 — every page is linked from a hub at or above its directory
inbound_from_hub: dict[str, bool] = {rel: False for rel in self.pages}
for rel in self.pages:
hub = self._hub_for(rel)
if hub is None:
inbound_from_hub[rel] = True
continue
inbound_from_hub[rel] = rel in graph.get(hub, set()) or hub == rel
for rel, ok in sorted(inbound_from_hub.items()):
if ok or not urlmap.in_sitemap(rel):
continue
self.gate("link.hub_gap", rel,
f"section hub {self._hub_for(rel)} does not link to it (rule 54)")
# rules 52, 53 — warnings
inbound = collections.Counter()
for rel, outs in graph.items():
for t in outs:
inbound[t] += 1
for rel in sorted(self.pages):
if pagedata.entry(rel)["kind"] in pagedata.ARTICLE_KINDS:
if len(graph[rel]) < 3:
self.warn("link.few_outbound", rel,
f"{len(graph[rel])} outbound internal links (<3) (rule 52)")
if inbound[rel] < 1:
self.warn("link.no_inbound", rel, "no inbound internal link (rule 52)")
for rel, hits in sorted(anchor_text.items()):
for text, snippet in hits:
self.warn("link.generic_anchor", rel,
f"anchor text {text!r} in {snippet!r} (rule 53)")
def _hub_for(self, rel: str) -> str | None:
"""The nearest hub page at or above `rel`'s directory."""
parts = rel.split("/")[:-1]
while parts:
d = "/".join(parts)
for cand in (f"{d}/index.html", f"{d}.html"):
if cand in self.pages and cand != rel:
return cand
parts.pop()
return "index.html" if rel != "index.html" else None
def _resolve(self, path: str, src_dir: pathlib.PurePosixPath) -> str | None:
"""Map an internal href to a repo-relative file, or None if it 404s."""
if path.startswith("/"):
base = path.lstrip("/")