122 lines
4.8 KiB
Bash
Executable File
122 lines
4.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# help4bis-site-scan.sh - Daily scan for help4bis.com
|
|
# Writes results to reports/SITE-SCAN.md in help4bis/help4bis-com Gitea repo
|
|
# Version: 2.0.0 | Created: 26/02/2026 | Updated: 23/10/2026
|
|
#
|
|
# Cron: 15 6 * * * /home/help4bis/scripts/help4bis-site-scan.sh
|
|
|
|
set -eo pipefail
|
|
|
|
set +u
|
|
source /home/help4bis/.env
|
|
set -u
|
|
|
|
GITEA_API="https://gitea.help4bis.com/api/v1"
|
|
GITEA_REPO="help4bis/help4bis-com"
|
|
REPORT_PATH="reports/SITE-SCAN.md"
|
|
SITE_URL="https://help4bis.com"
|
|
SCAN_DATE=$(date +"%Y-%m-%d %H:%M AEST")
|
|
LOG="/tmp/help4bis-site-scan.log"
|
|
ISSUES_FILE="/tmp/help4bis-scan-issues.count"
|
|
|
|
exec > "$LOG" 2>&1
|
|
echo "=== Help4Bis.com Site Scan: $SCAN_DATE ==="
|
|
|
|
echo "0" > "$ISSUES_FILE"
|
|
|
|
check_url() {
|
|
local url="$1"
|
|
local label="$2"
|
|
local code
|
|
code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$url" 2>/dev/null || echo "000")
|
|
if [ "$code" = "200" ]; then
|
|
echo "| $label | $url | OK | $code |"
|
|
else
|
|
echo "| $label | $url | FAIL | $code |"
|
|
local cur; cur=$(cat "$ISSUES_FILE")
|
|
echo $((cur + 1)) > "$ISSUES_FILE"
|
|
fi
|
|
}
|
|
|
|
REPORT=""
|
|
|
|
# Header
|
|
REPORT+="# Site Scan Report — help4bis.com\n"
|
|
REPORT+="**Scan date:** $SCAN_DATE\n\n"
|
|
|
|
# Every URL in sitemap.xml, plus the stylesheet and every internal link/asset on each page
|
|
# (cockpit #168: the old WordPress-era list checked URLs that no longer exist and missed a
|
|
# page that lost its CSS and 12 links).
|
|
REPORT+="## Sitemap URLs\n\n"
|
|
REPORT+="| Page | URL | Status | HTTP |\n"
|
|
REPORT+="|------|-----|--------|------|\n"
|
|
|
|
SITEMAP_URLS=$(curl -skL --connect-timeout 10 --max-time 20 "${SITE_URL}/sitemap.xml" | grep -o '<loc>[^<]*</loc>' | sed 's/<[^>]*>//g')
|
|
if [ -z "$SITEMAP_URLS" ]; then
|
|
REPORT+="| Sitemap | ${SITE_URL}/sitemap.xml | **UNREADABLE** | 000 |\n"
|
|
cur=$(cat "$ISSUES_FILE"); echo $((cur + 1)) > "$ISSUES_FILE"
|
|
fi
|
|
BROKEN_LINKS=""
|
|
# Clean URLs served by .htaccess rewrites: relative paths break here but not on the .html file
|
|
CLEAN_URLS="${SITE_URL}/pricing/ ${SITE_URL}/contact/ ${SITE_URL}/managed-wordpress/ ${SITE_URL}/wordpress-care-plans/"
|
|
for url in $SITEMAP_URLS $CLEAN_URLS; do
|
|
REPORT+="$(check_url "$url" "$(basename "${url%/}")")\n"
|
|
html=$(curl -skL --connect-timeout 10 --max-time 20 "$url" 2>/dev/null || true)
|
|
# css must be linked and load, as a visitor resolves it
|
|
for ref in $(printf '%s' "$html" | grep -o '\(href\|src\)="[^"#?]*"' | sed 's/^[a-z]*="//; s/"$//' | grep -v '^\(https\?:\|mailto:\|tel:\|data:\)' | sort -u); do
|
|
case "$ref" in
|
|
/*) full="${SITE_URL}${ref}" ;;
|
|
*) case "$url" in */) full="${url}${ref}" ;; *) full="${url%/*}/${ref}" ;; esac ;;
|
|
esac
|
|
code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$full" 2>/dev/null) || code="000"
|
|
if [ "$code" != "200" ]; then
|
|
sleep 3
|
|
code=$(curl -skL -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 15 "$full" 2>/dev/null) || code="000"
|
|
fi
|
|
if [ "$code" != "200" ]; then
|
|
BROKEN_LINKS+="| $url | $ref | $code |\n"
|
|
cur=$(cat "$ISSUES_FILE"); echo $((cur + 1)) > "$ISSUES_FILE"
|
|
fi
|
|
done
|
|
done
|
|
|
|
REPORT+="\n## Broken links and assets on those pages\n\n"
|
|
if [ -z "$BROKEN_LINKS" ]; then
|
|
REPORT+="None found.\n"
|
|
else
|
|
REPORT+="| Page | Reference | HTTP |\n|------|-----------|------|\n$BROKEN_LINKS"
|
|
fi
|
|
|
|
REPORT+="\n## Summary\n\n"
|
|
ISSUES=$(cat "$ISSUES_FILE")
|
|
REPORT+="- **Issues found:** $ISSUES\n"
|
|
REPORT+="- **Scan completed:** $SCAN_DATE\n"
|
|
if [ "$ISSUES" -eq 0 ]; then
|
|
REPORT+="- **Overall:** All checks passed\n"
|
|
else
|
|
REPORT+="- **Overall:** $ISSUES issue(s) need attention\n"
|
|
fi
|
|
|
|
# Write to Gitea
|
|
echo "Writing report to Gitea..."
|
|
REPORT_B64=$(printf '%b' "$REPORT" | base64 -w0)
|
|
EXISTING=$(curl -sk -H "Authorization: token $GITEA_TOKEN" \
|
|
"$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" 2>/dev/null)
|
|
EXISTING_SHA=$(echo "$EXISTING" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('sha',''))" 2>/dev/null || echo "")
|
|
|
|
if [ -n "$EXISTING_SHA" ]; then
|
|
curl -sk -X PUT -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
|
"$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" \
|
|
-d "{\"content\":\"$REPORT_B64\",\"sha\":\"$EXISTING_SHA\",\"message\":\"chore: daily site scan $(date +%Y-%m-%d)\"}" > /dev/null
|
|
echo "Updated existing report"
|
|
else
|
|
curl -sk -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
|
"$GITEA_API/repos/$GITEA_REPO/contents/$REPORT_PATH" \
|
|
-d "{\"content\":\"$REPORT_B64\",\"message\":\"feat: initial site scan $(date +%Y-%m-%d)\"}" > /dev/null
|
|
echo "Created new report"
|
|
fi
|
|
|
|
ISSUES=$(cat "$ISSUES_FILE")
|
|
echo "=== Scan complete. Issues: $ISSUES ==="
|
|
rm -f "$ISSUES_FILE"
|