#!/usr/bin/env python3 """Check the built Pages artifact, including URLs under the project base path.""" import argparse from html.parser import HTMLParser import json from pathlib import Path from urllib.parse import unquote, urljoin, urlsplit import xml.etree.ElementTree as ET BASE = "https://profundium.github.io/pg_local_cache/" class Page(HTMLParser): def __init__(self, text): super().__init__(convert_charrefs=True) self.ids, self.links, self.meta, self.copies = set(), [], {}, [] self.anchors = [] self.h1 = 0 self.canonical = None self.title = "" self.structured = [] self.errors = [] self.in_head = self.in_title = self.in_json = False self.json_text = "" self.feed(text) def handle_starttag(self, tag, attrs): attrs = dict(attrs) if "id" in attrs: if attrs["id"] in self.ids: self.errors.append(f"duplicate id: {attrs['id']}") self.ids.add(attrs["id"]) if tag == "h1": self.h1 += 1 if tag == "head": self.in_head = True if tag == "title" and self.in_head: self.in_title = True if tag == "meta": self.meta[attrs.get("name", attrs.get("property"))] = attrs.get("content", "") if tag == "link" and attrs.get("rel") == "canonical": if self.canonical is not None: self.errors.append("duplicate canonical") self.canonical = attrs.get("href") if "href" in attrs: self.links.append(attrs["href"]) if tag == "a": self.anchors.append(attrs["href"]) if "src" in attrs: self.links.append(attrs["src"]) if "data-copy" in attrs: self.copies.append(attrs["data-copy"]) if tag == "script" and attrs.get("type") == "application/ld+json": self.in_json, self.json_text = True, "" def handle_data(self, text): if self.in_title: self.title += text if self.in_json: self.json_text += text def handle_endtag(self, tag): if tag == "head": self.in_head = False if tag == "title": self.in_title = False if tag == "script" and self.in_json: try: self.structured.append(json.loads(self.json_text)) except ValueError as error: self.errors.append(f"invalid JSON-LD: {error}") self.in_json = False def check(root, base=BASE): base = base.rstrip("/") + "/" errors, pages, titles, descriptions = [], {}, set(), set() for path in root.rglob("*.html"): relative = path.relative_to(root).as_posix() url = base + (relative[:-10] if relative.endswith("index.html") else relative) page = Page(path.read_text()) pages[url] = page for error in page.errors: errors.append(f"{relative}: {error}") if page.h1 != 1 or "main-content" not in page.ids: errors.append(f"{relative}: expected one h1 and main-content") if page.canonical != url: errors.append(f"{relative}: incorrect canonical: {page.canonical}") if "noindex" in page.meta.get("robots", ""): errors.append(f"{relative}: unexpected indexing policy") description = page.meta.get("description", "") if not page.title.strip() or page.title in titles: errors.append(f"{relative}: missing or duplicate title") if not description or description in descriptions: errors.append(f"{relative}: missing or duplicate description") titles.add(page.title) descriptions.add(description) if not page.structured or page.meta.get("og:url") != url: errors.append(f"{relative}: missing structured data or wrong og:url") for prefix in ('og', 'twitter'): if page.meta.get(f'{prefix}:title') != page.title or page.meta.get(f'{prefix}:description') != description: errors.append(f"{relative}: inconsistent {prefix} title or description") image = page.meta.get(f'{prefix}:image', '') if not image.startswith('https://'): errors.append(f"{relative}: missing absolute {prefix} image URL") else: page.links.append(image) for target in page.copies: if target not in page.ids: errors.append(f"{relative}: copy target not found: {target}") if not pages: errors.append("no HTML pages were built") reachable, pending = set(), [base] while pending: url = pending.pop() if url in reachable or url not in pages: continue reachable.add(url) for target in pages[url].anchors: pending.append(urlsplit(urljoin(url, target))._replace(query="", fragment="").geturl()) for url in sorted(pages.keys() - reachable): errors.append(f"{url}: page is not reachable through links from the homepage") for url, page in pages.items(): for target in page.links: absolute = urljoin(url, target) parsed = urlsplit(absolute) if parsed.netloc != urlsplit(base).netloc: continue path_url = parsed._replace(query="", fragment="").geturl() if not path_url.startswith(base): errors.append(f"{url}: link escapes baseurl: {target}") continue relative = unquote(path_url[len(base):]) file = root / (relative + "index.html" if not relative or relative.endswith("/") else relative) if not file.is_file(): errors.append(f"{url}: missing local target: {target}") if parsed.fragment and path_url in pages and unquote(parsed.fragment) not in pages[path_url].ids: errors.append(f"{url}: missing fragment: {target}") try: sitemap = ET.parse(root / "sitemap.xml") urls = [node.text for node in sitemap.findall(".//{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] expected = {url for url, page in pages.items() if "noindex" not in page.meta.get("robots", "")} if len(urls) != len(set(urls)) or set(urls) != expected: errors.append("sitemap does not match the indexable canonical pages") except (OSError, ET.ParseError) as error: errors.append(f"invalid sitemap: {error}") return errors if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("directory", type=Path) parser.add_argument("--base-url", default=BASE) args = parser.parse_args() failures = check(args.directory, args.base_url) if failures: parser.exit(1, "\n".join(failures) + "\n") print("PASS: metadata, JSON-LD, sitemap, crawlable pages, links, fragments and copy targets")