chore: sitemap-driven site scan 2.0.0, copy of /home/help4bis/scripts (cockpit #168)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-23 06:45:43 +10:00
parent 3ea2be5a43
commit c91733764b
1 changed files with 121 additions and 0 deletions
@@ -0,0 +1,121 @@
+#!/bin/bash
+# help4bis-site-scan.sh - Daily scan for help4bis.com
+# Writes results to reports/SITE-SCAN.md in help4bis/help4bis-com Gitea repo
+# Version: 2.0.0 | Created: 26/02/2026 | Updated: 23/10/2026
+#
+# Cron: 15 6 * * * /home/help4bis/scripts/help4bis-site-scan.sh
+
+set -eo pipefail
+
+set +u
+source /home/help4bis/.env
+set -u
+
+GITEA_API="https://gitea.help4bis.com/api/v1"
+GITEA_REPO="help4bis/help4bis-com"
+REPORT_PATH="reports/SITE-SCAN.md"
+SITE_URL="https://help4bis.com"
+SCAN_DATE=$(date +"%Y-%m-%d %H:%M AEST")
+LOG="/tmp/help4bis-site-scan.log"
+ISSUES_FILE="/tmp/help4bis-scan-issues.count"
+
+exec > "$LOG" 2>&1
+echo "=== Help4Bis.com Site Scan: $SCAN_DATE ==="
+
+echo "0" > "$ISSUES_FILE"
+
+check_url() {
+    local url="$1"
+    local label="$2"
+    local code
+    code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$url" 2>/dev/null || echo "000")
+    if [ "$code" = "200" ]; then
+        echo "| $label | $url | OK | $code |"
+    else
+        echo "| $label | $url | FAIL | $code |"
+        local cur; cur=$(cat "$ISSUES_FILE")
+        echo $((cur + 1)) > "$ISSUES_FILE"
+    fi
+}
+
+REPORT=""
+
+# Header
+REPORT+="# Site Scan Report — help4bis.com\n"
+REPORT+="**Scan date:** $SCAN_DATE\n\n"
+
+# Every URL in sitemap.xml, plus the stylesheet and every internal link/asset on each page
+# (cockpit #168: the old WordPress-era list checked URLs that no longer exist and missed a
+# page that lost its CSS and 12 links).
+REPORT+="## Sitemap URLs\n\n"
+REPORT+="| Page | URL | Status | HTTP |\n"
+REPORT+="|------|-----|--------|------|\n"
+
+SITEMAP_URLS=$(curl -skL --connect-timeout 10 --max-time 20 "${SITE_URL}/sitemap.xml" | grep -o '<loc>[^<]*</loc>' | sed 's/<[^>]*>//g')
+if [ -z "$SITEMAP_URLS" ]; then
+    REPORT+="| Sitemap | ${SITE_URL}/sitemap.xml | **UNREADABLE** | 000 |\n"
+    cur=$(cat "$ISSUES_FILE"); echo $((cur + 1)) > "$ISSUES_FILE"
+fi
+BROKEN_LINKS=""
+# Clean URLs served by .htaccess rewrites: relative paths break here but not on the .html file
+CLEAN_URLS="${SITE_URL}/pricing/ ${SITE_URL}/contact/"
+for url in $SITEMAP_URLS $CLEAN_URLS; do
+    REPORT+="$(check_url "$url" "$(basename "${url%/}")")\n"
+    html=$(curl -skL --connect-timeout 10 --max-time 20 "$url" 2>/dev/null || true)
+    # css must be linked and load, as a visitor resolves it
+    for ref in $(printf '%s' "$html" | grep -o '\(href\|src\)="[^"#?]*"' | sed 's/^[a-z]*="//; s/"$//' | grep -v '^\(https\?:\|mailto:\|tel:\|data:\)' | sort -u); do
+        case "$ref" in
+            /*) full="${SITE_URL}${ref}" ;;
+            *)  case "$url" in */) full="${url}${ref}" ;; *) full="${url%/*}/${ref}" ;; esac ;;
+        esac
+        code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$full" 2>/dev/null) || code="000"
+        if [ "$code" != "200" ]; then
+            sleep 3
+            code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$full" 2>/dev/null) || code="000"
+        fi
+        if [ "$code" != "200" ]; then
+            BROKEN_LINKS+="| $url | $ref | $code |\n"
+            cur=$(cat "$ISSUES_FILE"); echo $((cur + 1)) > "$ISSUES_FILE"
+        fi
+    done
+done
+
+REPORT+="\n## Broken links and assets on those pages\n\n"
+if [ -z "$BROKEN_LINKS" ]; then
+    REPORT+="None found.\n"
+else
+    REPORT+="| Page | Reference | HTTP |\n|------|-----------|------|\n$BROKEN_LINKS"
+fi
+
+REPORT+="\n## Summary\n\n"
+ISSUES=$(cat "$ISSUES_FILE")
+REPORT+="- **Issues found:** $ISSUES\n"
+REPORT+="- **Scan completed:** $SCAN_DATE\n"
+if [ "$ISSUES" -eq 0 ]; then
+    REPORT+="- **Overall:** All checks passed\n"
+else
+    REPORT+="- **Overall:** $ISSUES issue(s) need attention\n"
+fi
+
+# Write to Gitea
+echo "Writing report to Gitea..."
+REPORT_B64=$(printf '%b' "$REPORT" | base64 -w0)
+EXISTING=$(curl -sk -H "Authorization: token $GITEA_TOKEN" \
+    "$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" 2>/dev/null)
+EXISTING_SHA=$(echo "$EXISTING" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('sha',''))" 2>/dev/null || echo "")
+
+if [ -n "$EXISTING_SHA" ]; then
+    curl -sk -X PUT -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+        "$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" \
+        -d "{\"content\":\"$REPORT_B64\",\"sha\":\"$EXISTING_SHA\",\"message\":\"chore: daily site scan $(date +%Y-%m-%d)\"}" > /dev/null
+    echo "Updated existing report"
+else
+    curl -sk -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+        "$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" \
+        -d "{\"content\":\"$REPORT_B64\",\"message\":\"feat: initial site scan $(date +%Y-%m-%d)\"}" > /dev/null
+    echo "Created new report"
+fi
+
+ISSUES=$(cat "$ISSUES_FILE")
+echo "=== Scan complete. Issues: $ISSUES ==="
+rm -f "$ISSUES_FILE"