add sitemap generation and automated IndexNow integration

Author: thestinger

deploy-static

@@ -16,10 +16,16 @@ [email protected]
 
 # use last modified timestamps from attestation.app
 rsync -rptcv --chmod=D755,F644 --delete --fsync --preallocate $remote:/srv/attestation.app/ static-production
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-production/sitemap.xml{,.gz,.br} static-tmp/
 rsync -rpcv --chmod=D755,F644 --delete --fsync --preallocate static-tmp/ static-production
 for f in static-production/**.*(br|gz); do
     touch -r "${f%.*}" "$f"
 done
+changed="$(./generate-sitemap)"
+xmllint --noblanks static-tmp/sitemap.xml --output static-tmp/sitemap.xml
+brotli -f static-tmp/sitemap.xml
+zopfli static-tmp/sitemap.xml
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-tmp/sitemap.xml{,.gz,.br} static-production/
 
 active=$(ssh $remote readlink /srv/attestation.app)
 
@@ -48,3 +54,7 @@ rsync -pcv --chmod=755 --fsync --preallocate remote-backup $remote:/usr/local/bi
 rsync -pcv --chmod=644 --fsync --preallocate systemd/system/remote-backup.timer $remote:/etc/systemd/system/remote-backup.timer
 rsync -pcv --chmod=644 --fsync --preallocate systemd/system/remote-backup.service $remote:/etc/systemd/system/remote-backup.service
 rsync -pcv --chmod=644 --chown attestation:attestation --fsync --preallocate backup-public-key.txt cloud-archive.sh $remote:/var/lib/attestation/
+
+if [[ -n "$changed" ]]; then
+    ./indexnow <<< "$changed"
+fi

generate-sitemap

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+from datetime import datetime, timezone
+from os.path import getmtime
+from pathlib import Path
+
+base = "https://attestation.app"
+
+pages = [
+    ["/", 1.0],
+    ["/.well-known/security.txt", 0.0],
+    ["/LICENSE.txt", 0.0],
+    ["/about", 1.0],
+    ["/donate", 0.5],
+    ["/contact", 0.1],
+    ["/humans.txt", 0.0],
+    ["/privacy-policy", 0.2],
+    ["/source", 0.1],
+    ["/tutorial", 1.0]
+]
+
+base_mtime = getmtime("static-tmp")
+entries = []
+
+for page in pages:
+    path = page[0]
+    loc = base + path
+    filepath = "static-production" + path
+    if path[-1] == '/':
+        filepath += "index.html"
+    elif "." not in path:
+        filepath += ".html"
+
+    mtime = getmtime(filepath)
+    if mtime > base_mtime:
+        print(loc)
+    lastmod = datetime.fromtimestamp(mtime, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%:z")
+    priority = page[1]
+    entries.append(f"""
+    <url>
+        <loc>{loc}</loc>
+        <lastmod>{lastmod}</lastmod>
+        <priority>{priority}</priority>
+    </url>""")
+
+sitemap = f"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">{"".join(entries)}
+</urlset>
+"""
+
+with open("static-tmp/sitemap.xml", "w") as f:
+    f.write(sitemap)