Add sso-diagnose.sh: verify all peertube-sso-incident claims into a log

- docs: record the open identity-forwarding question, auth contracts, and responsibility boundary

scripts/echo-headers.php

@@ -0,0 +1,51 @@
+<?php
+/**
+ * echo-headers.php — request-side identity probe for the SSO diagnosis.
+ *
+ * The ONLY reliable way to see the identity header LemonLDAP injects into the
+ * request IT forwards to the backend is to place an endpoint *behind* the same
+ * reverse proxy (same vhost as PeerTube) and have it echo the request headers.
+ * Response headers on the public site can never reveal this.
+ *
+ * Deploy this file anywhere served through the SAME LemonLDAP vhost that
+ * protects videos.erg.be (e.g. a static location, or a tiny PHP handler on the
+ * backend), authenticate at portail.erg.school, then hit it — it returns JSON
+ * of every inbound header. Forward the resulting URL to sso-diagnose.sh:
+ *
+ *   scripts/sso-diagnose.sh --echo 'https://videos.erg.be/path/to/echo-headers.php'
+ *
+ * Output (subset):
+ *   {
+ *     "headers": { "X-Remote-User": "jsmith", "Auth-User": "jsmith", ... },
+ *     "server":  { "REMOTE_USER": "...", ... }
+ *   }
+ *
+ * It also mirrors the CGI subprocess environment (REMOTE_USER, etc.), which is
+ * where Apache/LemonLDAP often land the identity. Safe: emits NO secret — only
+ * the incoming headers it was handed.
+ */
+
+header('Content-Type: application/json; charset=utf-8');
+header('Cache-Control: no-store');
+
+$headers = [];
+foreach ($_SERVER as $k => $v) {
+    if (str_starts_with($k, 'HTTP_')) {
+        $name = str_replace('_', '-', substr($k, 5));
+        $headers[$name] = $v;
+    }
+}
+
+// Also surface the classic CGI environment identity vars directly.
+$server = [
+    'REMOTE_USER'       => $_SERVER['REMOTE_USER'] ?? null,
+    'AUTH_USER'         => $_SERVER['AUTH_USER'] ?? null,
+    'PHP_AUTH_USER'     => $_SERVER['PHP_AUTH_USER'] ?? null,
+    'REDIRECT_REMOTE_USER' => $_SERVER['REDIRECT_REMOTE_USER'] ?? null,
+];
+
+echo json_encode([
+    'headers' => $headers,
+    'server'  => $server,
+    'remote_addr' => $_SERVER['REMOTE_ADDR'] ?? null,
+], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);

scripts/sso-diagnose.sh

@@ -0,0 +1,291 @@
+#!/usr/bin/env bash
+# =============================================================================
+# sso-diagnose.sh — verify every claim in docs/peertube-sso-incident.md
+#
+# Checks the topology + auth behaviour end-to-end and appends a timestamped,
+# machine-parseable block to a log for later analysis. NO secret is written to
+# the log or stdout: SMTP AUTH is tested (proves the credential is still valid)
+# but the password is never echoed, and any PeerTube token/secret is redacted.
+#
+# What it verifies (mirrors the incident report):
+#   1. DNS + reverse-DNS + HTTP identity of the three hosts
+#      (videos.erg.be, mail.erg.school, portail.erg.school)
+#   2. The SSO IdP = LemonLDAP::NG OIDC, and its discovery document
+#      (grant/response types — no `password`, no `client_credentials`)
+#   3. PeerTube's OAuth clients (the built-in `local` client) and the
+#      `password` grant result (`invalid_grant` expected)
+#   4. SMTP STARTTLS AUTH mechanisms (XOAUTH2/OAUTHBEARER vs PLAIN/LOGIN)
+#   5. PreserveHost / identity-header propagation between the proxy and backend
+#      (REMOTE_USER / X-Remote-User / X-Forwarded-User / Auth-User headers)
+#      — response-side + an optional request-side echo probe (--echo <url>) that
+#      uses scripts/echo-headers.php to reveal headers the SSO proxy injects.
+#
+# Usage:
+#   scripts/sso-diagnose.sh [--instance <url>] [--idp <url>] [--smtp <host:port>]
+#                           [--mail-host <host>] [--db <path>] [--log <path>]
+#                           [--echo <url>]
+#
+# Exit 0 always (a diagnostic records results); individual findings carry
+# PASS/FAIL/INFO in the log so analysis is unambiguous.
+#
+# NOTE: SMTP creds + password come from the xamxam DB ONLY if creds-probe.php
+# exists and can run; the plaintext password is NEVER printed — only the fact
+# that AUTH succeeded/failed. If you want PeerTube's password-grant probe to run
+# you must run `just creds-test` (it needs the password to make the request).
+# =============================================================================
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# ── defaults ───────────────────────────────────────────────────────────────────
+INSTANCE="${INSTANCE:-https://videos.erg.be}"
+IDP_HOST="${IDP_HOST:-portail.erg.school}"
+SMTP_HOST="${SMTP_HOST:-mail.erg.school}"
+SMTP_PORT="${SMTP_PORT:-587}"
+DB_PATH="${DB_PATH:-$REPO_ROOT/app/storage/xamxam.db}"
+LOG="${LOG:-$REPO_ROOT/sso-diagnose.log}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --instance)  INSTANCE="$2";  shift 2 ;;
+    --idp)       IDP_HOST="$2";  shift 2 ;;
+    --smtp)      SMTP_HOST="$2"; shift 2 ;;
+    --mail-host) SMTP_HOST="$2"; shift 2 ;;
+    --db)        DB_PATH="$2";   shift 2 ;;
+    --log)       LOG="$2";       shift 2 ;;
+    --echo)      ECHO_URL="$2";  shift 2 ;;
+    -h|--help)
+      echo "Usage: scripts/sso-diagnose.sh [--instance <url>] [--idp <host>] [--smtp <host:port>] [--db <path>] [--log <path>] [--echo <url>]"
+      exit 0 ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+IDP_URL="https://${IDP_HOST}"
+
+# ── log plumbing (machine-parseable) ──────────────────────────────────────────
+log()      { printf '%s\n' "$*" | tee -a "$LOG"; }
+section()  { log "  ── $*"; }
+kv()       { log "  %-28s %s" "$1:" "$2"; }
+
+# redact anything that looks like a secret/token from a value before logging
+redact() {
+  local v="$1"
+  v="${v//$'\n'/ | }"
+  # strip obvious bearer/oauth secrets & base64-ish blobs
+  sed -E 's/(["'\''A-Za-z0-9_.\/+-]{24,})/<REDACTED>/g' <<<"$v"
+}
+
+log ""
+log "════════════════════════════════════════════════════════════════════"
+log "XAMXAM SSO/PeerTube diagnosis — $(date '+%Y-%m-%d %H:%M:%S %z')"
+log "════════════════════════════════════════════════════════════════════"
+
+# ── preflight ─────────────────────────────────────────────────────────────────
+for c in curl jq dig host; do
+  command -v "$c" >/dev/null || { log "  MISSING-DEP $c"; }
+done
+
+RESOLVE_TOOL="getent"
+command -v getent >/dev/null || RESOLVE_TOOL="host"
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 1. DNS + reverse DNS + HTTP identity  (incident §3.5)
+# ═══════════════════════════════════════════════════════════════════════════════
+section "topology / DNS / reverse-DNS / HTTP identity"
+
+hosts=("videos.erg.be" "$SMTP_HOST" "$IDP_HOST")
+for h in "${hosts[@]}"; do
+  if [[ "$RESOLVE_TOOL" == "getent" ]]; then
+    ips="$(getent ahostsv4 "$h" 2>/dev/null | awk '{print $1}' | sort -u | head -3 | tr '\n' ' ')"
+  else
+    ips="$(host -t A "$h" 2>/dev/null | awk '/has address/{print $4}' | sort -u | head -3 | tr '\n' ' ')"
+  fi
+  ips="${ips// /}"
+  if [[ -z "$ips" ]]; then
+    kv "DNS[$h]" "NO-A-RECORD"
+    continue
+  fi
+  kv "DNS[$h]" "$(echo "$ips" | tr ' ' ',')"
+  first_ip="$(echo "$ips" | awk '{print $1}')"
+
+  # reverse DNS
+  if command -v dig >/dev/null; then
+    rdns="$(dig +short -x "$first_ip" 2>/dev/null | tr '\n' ' ' | sed 's/[[:space:]]*$//')"
+  else
+    rdns="$(host "$first_ip" 2>/dev/null | awk '/pointer/{print $NF}' | sed 's/\.$//')"
+  fi
+  kv "rDNS[$first_ip]" "$(redact "${rdns:-none}")"
+done
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 2. SSO IdP discovery document  (incident §3.5 — decisive fact)
+# ═══════════════════════════════════════════════════════════════════════════════
+section "SSO IdP discovery ($IDP_URL/.well-known/openid-configuration)"
+
+disc="$(curl -sS -m 20 -w $'\n%{http_code}' "$IDP_URL/.well-known/openid-configuration" 2>/dev/null)"
+disc_code="${disc##*$'\n'}"
+disc_body="${disc%$'\n'*}"
+disc_body="${disc_body%$'\n'}"
+
+kv "HTTP" "$disc_code"
+if [[ "$disc_code" == "200" ]]; then
+  issuer="$(echo "$disc_body" | jq -r '.issuer // "n/a"')"
+  grants="$(echo "$disc_body" | jq -r '.grant_types_supported // [] | join(",")')"
+  resp_types="$(echo "$disc_body" | jq -r '.response_types_supported // [] | join(",")')"
+  auth_ep="$(echo "$disc_body" | jq -r '.authorization_endpoint // "n/a"')"
+  token_ep="$(echo "$disc_body" | jq -r '.token_endpoint // "n/a"')"
+  kv "issuer" "$issuer"
+  kv "grant_types_supported" "$grants"
+  kv "response_types_supported" "$resp_types"
+  kv "authorization_endpoint" "$auth_ep"
+  kv "token_endpoint" "$token_ep"
+
+  # decisive checks
+  has_password=false; has_cc=false; has_authcode=false
+  echo "$grants" | grep -q 'password'             && has_password=true
+  echo "$grants" | grep -q 'client_credentials'   && has_cc=true
+  echo "$grants" | grep -q 'authorization_code'   && has_authcode=true
+
+  log "  FINDING grant.password             → $([[ $has_password == true ]] && echo PRESENT || echo ABSENT)"
+  log "  FINDING grant.client_credentials   → $([[ $has_cc == true ]] && echo PRESENT || echo ABSENT)"
+  log "  FINDING grant.authorization_code   → $([[ $has_authcode == true ]] && echo PRESENT || echo ABSENT)"
+
+  if [[ $has_password == false && $has_cc == false ]]; then
+    log "  CONCLUSION: IdP offers NO non-interactive grant → password/app-token paths are dead (matches report)."
+  fi
+fi
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 3. PeerTube OAuth clients + password grant  (incident §1 facts 3–5)
+# ═══════════════════════════════════════════════════════════════════════════════
+section "PeerTube OAuth ($INSTANCE)"
+
+api="${INSTANCE%/}/api/v1"
+ptcode="$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$INSTANCE/api/v1/config" 2>/dev/null)"
+kv "config HTTP" "$ptcode"
+
+# built-in local client — its client_id is public, its secret is NOT logged
+local_client="$(curl -sS -m 20 "$api/oauth-clients/local" 2>/dev/null)"
+if echo "$local_client" | jq -e '.client_id' >/dev/null 2>&1; then
+  cid="$(echo "$local_client" | jq -r '.client_id')"
+  kv "local client_id" "$(redact "$cid")"
+  kv "local client present" "yes"
+else
+  kv "local client" "not returned (may need auth)"
+fi
+
+# password grant result (only if creds-probe.php can supply creds securely)
+if [[ -f "$DB_PATH" && -f "$SCRIPT_DIR/creds-probe.php" && -x "$SCRIPT_DIR/creds-probe.php" ]]; then
+  probe_out="$(php "$SCRIPT_DIR/creds-probe.php" --db "$DB_PATH" --instance "$INSTANCE" 2>/dev/null)"
+  # $argv/$j are PHP variables (not shell); single quotes are intentional.
+  # shellcheck disable=SC2016
+  pt_ok="$(php -r 'echo (json_decode($argv[1],true)["peertube"]["ok"]??false)?"1":"0";' "$probe_out" 2>/dev/null)"
+  # shellcheck disable=SC2016
+  pt_code="$(php -r '$j=json_decode($argv[1],true)["peertube"]??[]; echo $j["code"]??"";' "$probe_out" 2>/dev/null)"
+  kv "password-grant ok" "$pt_ok"
+  kv "password-grant code" "$(redact "$pt_code")"
+  if [[ "$pt_ok" == "0" && -n "$pt_code" ]]; then
+    log "  FINDING password-grant → REJECTED ($pt_code) — matches 'invalid_grant' if '$pt_code' == 'invalid_grant'"
+  fi
+else
+  kv "password-grant" "SKIPPED (creds-probe.php or DB unavailable → run: just creds-test)"
+fi
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 4. SMTP STARTTLS AUTH mechanisms  (incident §3.5 — why SMTP still works)
+# ═══════════════════════════════════════════════════════════════════════════════
+section "SMTP AUTH mechanisms ($SMTP_HOST:$SMTP_PORT)"
+
+smtp_out="$(timeout 25 python3 - "$SMTP_HOST" "$SMTP_PORT" 2>/dev/null <<'PYEOF'
+import smtplib, ssl, sys
+host, port = sys.argv[1], int(sys.argv[2])
+try:
+    s = smtplib.SMTP(host, port, timeout=20)
+    s.ehlo()
+    s.starttls(context=ssl.create_default_context())
+    s.ehlo()
+    print(s.esmtp_features.get("auth", "NONE"))
+    s.quit()
+except Exception as e:
+    print("ERROR: %s" % e)
+PYEOF
+)"
+kv "AUTH mechanisms" "$(redact "$smtp_out")"
+
+if echo "$smtp_out" | grep -qiE 'XOAUTH2|OAUTHBEARER'; then
+  log "  FINDING SMTP → SSO/OAuth2 wired in (XOAUTH2/OAUTHBEARER present)"
+fi
+if echo "$smtp_out" | grep -qiE '(^| )(PLAIN|LOGIN)( |$)'; then
+  log "  FINDING SMTP → legacy PLAIN/LOGIN KEPT → app's PLAIN auth still works (matches report)"
+else
+  log "  FINDING SMTP → PLAIN/LOGIN absent → SMTP would ALSO be broken"
+fi
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 5. Identity-header / PreserveHost propagation  (the diagnosis' core question)
+# ═══════════════════════════════════════════════════════════════════════════════
+section "proxy → backend identity-header propagation"
+
+# (a) RESPONSE headers — what the public endpoint reflects. This can only ever show
+# *response* headers (X-Powered-By, Server, ...), NEVER an identity header that
+# LemonLDAP injects INTO the request. Absence here is NOT proof of absence upstream.
+kv "response-header scope" "response side only — cannot reveal inbound identity"
+hdrs="$(curl -sS -m 20 -D - -o /dev/null "$INSTANCE/" 2>/dev/null)"
+for hname in X-Powered-By Server Removed-User; do
+  val="$(printf '%s' "$hdrs" | grep -i "^${hname}:" | head -1 | sed "s/^[^:]*:[[:space:]]*//" | tr -d '\r')"
+  if [[ -n "$val" ]]; then
+    kv "response [$hname]" "$(redact "$val")"
+  fi
+done
+
+# (b) REQUEST-side identity headers — the part that actually answers the "❓".
+# These headers are injected by the SSO reverse-proxy into the request it forwards
+# to the backend. They are NOT visible in any public response; the ONLY way to see
+# them is to have an endpoint *behind* the proxy echo the request headers back.
+#
+# Two ways to get that echo:
+#   --  self-host: run `scripts/echo-headers.php` behind the SAME LemonLDAP vhost
+#       as PeerTube and point --echo at it, OR
+#   --  a known public echo service (httpbin.org/headers) — but that only shows
+#       headers the PUBLIC client sent, not ones LemonLDAP adds AFTER auth.
+section "request-side identity header probe (echo endpoint)"
+
+ECHO_URL="${ECHO_URL:-}"
+if [[ -n "$ECHO_URL" ]]; then
+  echo_body="$(curl -sS -m 20 "$ECHO_URL" 2>/dev/null)"
+  if echo "$echo_body" | jq -e '.headers' >/dev/null 2>&1; then
+    # extract any identity-ish header, case-insensitively
+    identity_hdrs="$(echo "$echo_body" | jq -r '.headers | to_entries[] | select(.key | test("remote[-_]?user|auth[-_]?user|x[-_]?forwarded[-_]?user|x[-_]?user|proxy[-_]?user|oidc[-_]?claim"; "i")) | "\(.key)=\(.value)"')"
+    if [[ -n "$identity_hdrs" ]]; then
+      log "  → IDENTITY HEADERS SEEN AT BACKEND:"
+      while IFS= read -r l; do kv "  identity" "$(redact "$l")"; done <<< "$identity_hdrs"
+      log "  CONCLUSION: an authenticated-user header IS reaching the backend."
+    else
+      log "  → NO identity header in the echoed request headers."
+      log "  CONCLUSION: LemonLDAP is protecting PeerTube but NOT telling it WHO the user is"
+      log "             → PeerTube has no authenticated identity to act on."
+    fi
+  else
+    kv "echo endpoint" "unparseable/non-JSON (got: $(redact "${echo_body:0:80}"))"
+  fi
+elif [[ -f "$SCRIPT_DIR/echo-headers.php" ]]; then
+  kv "echo endpoint" "SKIPPED — pass --echo <url> (point it at echo-headers.php behind the same vhost)"
+else
+  kv "echo endpoint" "NOT AVAILABLE — see scripts/echo-headers.php docs; --echo <url> enables the probe"
+fi
+log "  HINT: to see headers LemonLDAP injects, host scripts/echo-headers.php behind"
+log "        the SAME LemonLDAP vhost as PeerTube and re-run with --echo <that-url>."
+log "        A public echo service only shows headers YOUR client sent, not the"
+log "        post-auth headers the SSO proxy adds."
+
+# whether ProxyPreserveHost-equivalent is observable: compare Host saw vs sent
+host_sent="$(curl -sS -m 20 -s -o /dev/null -w '%{url_effective}' "$INSTANCE/" 2>/dev/null)"
+kv "effective-url" "$host_sent"
+
+log ""
+log "════════════════════════════════════════════════════════════════════"
+log "summary → log written to: $LOG"
+log "════════════════════════════════════════════════════════════════════"
+exit 0