163 lines
6.8 KiB
Bash
Executable File
163 lines
6.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# fleet-log-check: Autonomous daily log auditor and journal aggregator for t520, bazina, and boxC
|
|
set -euo pipefail
|
|
|
|
TODAY=$(date +%Y-%m-%d)
|
|
NOW_TS=$(date +%H:%M:%S)
|
|
ARCHIVE_DIR="$HOME/src/journals-fleet"
|
|
AUDIT_FILE="$ARCHIVE_DIR/system-audits/${TODAY}.md"
|
|
DNES_DIR="$HOME/src/dnes"
|
|
LOGS_DIR="$DNES_DIR/logs"
|
|
|
|
mkdir -p "$ARCHIVE_DIR/system-audits" "$ARCHIVE_DIR/dnes" "$ARCHIVE_DIR/now" "$LOGS_DIR"
|
|
|
|
ALERT_COUNT=0
|
|
ALERTS=()
|
|
|
|
echo "=== Running Fleet Daily Audit for $TODAY ==="
|
|
|
|
# --- 1. AUDIT boxC ---
|
|
BOXC_HOST="boxC"
|
|
BOXC_ONLINE=0
|
|
BOXC_KOF17_AVAIL="N/A"
|
|
BOXC_KOF17_PCT="N/A"
|
|
BOXC_SYNCTHING="N/A"
|
|
BOXC_FAILED_UNITS=""
|
|
BOXC_RECENT_ERRORS=""
|
|
BOXC_SSH_BLOCKED=0
|
|
|
|
if ssh -o ConnectTimeout=3 "$BOXC_HOST" "true" 2>/dev/null; then
|
|
BOXC_ONLINE=1
|
|
# Check /mnt/kof17 disk space
|
|
read -r b_avail b_pct < <(ssh "$BOXC_HOST" "df -BG /mnt/kof17 | awk 'NR==2 {print \$4, \$5}'" 2>/dev/null || echo "0G 100%")
|
|
BOXC_KOF17_AVAIL="$b_avail"
|
|
BOXC_KOF17_PCT="$b_pct"
|
|
|
|
# Check threshold: if available < 3G or pct >= 98%
|
|
AVAIL_NUM=$(echo "$b_avail" | tr -dc '0-9' || echo 0)
|
|
if [ "$AVAIL_NUM" -le 2 ]; then
|
|
ALERT_COUNT=$((ALERT_COUNT + 1))
|
|
ALERTS+=("CRITICAL: boxC /mnt/kof17 is critically low on space ($BOXC_KOF17_AVAIL free, $BOXC_KOF17_PCT used)!")
|
|
fi
|
|
|
|
# Syncthing status & errors
|
|
BOXC_SYNCTHING=$(ssh "$BOXC_HOST" "systemctl --user is-active syncthing 2>&1" || echo "failed")
|
|
SYNCTHING_ERRS=$(ssh "$BOXC_HOST" "journalctl --user -u syncthing --since '24 hours ago' --no-pager 2>/dev/null | grep -E 'connection reset by peer|TLS handshake|FATAL|ERROR' | tail -n 5" || true)
|
|
|
|
# Failed units
|
|
BOXC_FAILED_UNITS=$(ssh "$BOXC_HOST" "systemctl --failed --no-legend 2>/dev/null; systemctl --user --failed --no-legend 2>/dev/null" || true)
|
|
if [ -n "$BOXC_FAILED_UNITS" ]; then
|
|
ALERT_COUNT=$((ALERT_COUNT + 1))
|
|
ALERTS+=("WARNING: boxC has failed systemd units: $BOXC_FAILED_UNITS")
|
|
fi
|
|
|
|
# Journal errors
|
|
BOXC_RECENT_ERRORS=$(ssh "$BOXC_HOST" "journalctl -p err..emerg --since '24 hours ago' --no-pager 2>/dev/null | tail -n 8" || true)
|
|
|
|
# SSH blocker count
|
|
BOXC_SSH_BLOCKED=$(ssh "$BOXC_HOST" "grep -c 'DROP' /tmp/ssh_blocked.log 2>/dev/null || echo 0" || true)
|
|
else
|
|
ALERT_COUNT=$((ALERT_COUNT + 1))
|
|
ALERTS+=("CRITICAL: boxC is UNREACHABLE over SSH!")
|
|
fi
|
|
|
|
# --- 2. AUDIT bazina ---
|
|
BAZINA_HOST="bazina"
|
|
BAZINA_ONLINE=0
|
|
BAZINA_SSD_AVAIL="N/A"
|
|
BAZINA_SEAGATE_AVAIL="N/A"
|
|
BAZINA_LOAD="N/A"
|
|
BAZINA_MEM="N/A"
|
|
BAZINA_OSC_REC="N/A"
|
|
|
|
if ssh -o ConnectTimeout=3 "$BAZINA_HOST" "true" 2>/dev/null; then
|
|
BAZINA_ONLINE=1
|
|
BAZINA_SSD_AVAIL=$(ssh "$BAZINA_HOST" "df -h /mnt/ssd 2>/dev/null | awk 'NR==2 {print \$4}'" || echo "N/A")
|
|
BAZINA_SEAGATE_AVAIL=$(ssh "$BAZINA_HOST" "df -h /mnt/seagate 2>/dev/null | awk 'NR==2 {print \$4}'" || echo "N/A")
|
|
BAZINA_LOAD=$(ssh "$BAZINA_HOST" "uptime | awk -F'load average:' '{print \$2}' | sed 's/ //g'" || echo "N/A")
|
|
BAZINA_MEM=$(ssh "$BAZINA_HOST" "free -h | awk '/Mem:/ {print \$3 \"/\" \$2}'" || echo "N/A")
|
|
BAZINA_OSC_REC=$(ssh "$BAZINA_HOST" "systemctl is-active osc-recorder.service 2>/dev/null" || echo "inactive")
|
|
else
|
|
ALERT_COUNT=$((ALERT_COUNT + 1))
|
|
ALERTS+=("CRITICAL: bazina is UNREACHABLE over SSH!")
|
|
fi
|
|
|
|
# --- 3. AUDIT t520 (Local) ---
|
|
T520_LOAD=$(uptime | awk -F'load average:' '{print $2}' | sed 's/ //g')
|
|
T520_MEM=$(free -h | awk '/Mem:/ {print $3 "/" $2}')
|
|
T520_ZRAM=$(zramctl --noheadings 2>/dev/null | awk '{print $1 " size:" $3 " used:" $4 " comp:" $6}' || echo "inactive")
|
|
CHRONY_DRIFT=$(chronyc tracking 2>/dev/null | awk '/Last offset/ {print $4 " " $5}' || echo "N/A")
|
|
CHRONY_STRATUM=$(chronyc tracking 2>/dev/null | awk '/Stratum/ {print $3}' || echo "N/A")
|
|
|
|
# --- 4. GENERATE MARKDOWN AUDIT REPORT ---
|
|
cat << EOF > "$AUDIT_FILE"
|
|
# Fleet System Audit: $TODAY ($NOW_TS CET)
|
|
|
|
## Health Summary
|
|
- **Overall Status**: $([ "$ALERT_COUNT" -eq 0 ] && echo "🟢 ALL SYSTEMS HEALTHY" || echo "🔴 $ALERT_COUNT ALERTS DETECTED")
|
|
- **Active Nodes**: t520 (local), bazina ($([ "$BAZINA_ONLINE" -eq 1 ] && echo "ONLINE" || echo "OFFLINE")), boxC ($([ "$BOXC_ONLINE" -eq 1 ] && echo "ONLINE" || echo "OFFLINE"))
|
|
|
|
$([ "$ALERT_COUNT" -gt 0 ] && echo "### Active Alerts" && for a in "${ALERTS[@]}"; do echo "- ⚠️ $a"; done || true)
|
|
|
|
---
|
|
|
|
## 1. Node Telemetry Matrix
|
|
|
|
| Node | Reachable | Memory (Used/Total) | Storage Status | Key Daemons / Swap | Load Avg |
|
|
|:---|:---|:---|:---|:---|:---|
|
|
| **t520** | Local | $T520_MEM | Root ext4 / zram0 ($T520_ZRAM) | Chrony Stratum $CHRONY_STRATUM (offset $CHRONY_DRIFT) | $T520_LOAD |
|
|
| **bazina** | $([ "$BAZINA_ONLINE" -eq 1 ] && echo "Yes" || echo "No") | $BAZINA_MEM | SSD: $BAZINA_SSD_AVAIL free \| Seagate: $BAZINA_SEAGATE_AVAIL | osc-recorder: $BAZINA_OSC_REC | $BAZINA_LOAD |
|
|
| **boxC** | $([ "$BOXC_ONLINE" -eq 1 ] && echo "Yes" || echo "No") | - | **/mnt/kof17: $BOXC_KOF17_AVAIL free ($BOXC_KOF17_PCT)** | Syncthing: $BOXC_SYNCTHING | - |
|
|
|
|
---
|
|
|
|
## 2. boxC Storage & Daemon Diagnostics
|
|
- **/mnt/kof17 Available**: $BOXC_KOF17_AVAIL (Threshold: alert if <= 2G)
|
|
- **Syncthing Unit**: $BOXC_SYNCTHING
|
|
$([ -n "${SYNCTHING_ERRS:-}" ] && echo "\`\`\`" && echo "$SYNCTHING_ERRS" && echo "\`\`\`" || echo "- Syncthing connection logs: Clean.")
|
|
|
|
### Systemd Failed Units (boxC):
|
|
$([ -n "$BOXC_FAILED_UNITS" ] && echo "$BOXC_FAILED_UNITS" || echo "None. All units operational.")
|
|
|
|
### Recent Critical/Error Logs (24h boxC):
|
|
$([ -n "$BOXC_RECENT_ERRORS" ] && echo "\`\`\`" && echo "$BOXC_RECENT_ERRORS" && echo "\`\`\`" || echo "None recorded.")
|
|
|
|
---
|
|
|
|
## 3. Fleet Synchronization Checkpoints
|
|
- Mirror target: \`bazina:/mnt/ssd/repos/journals-fleet.git\`
|
|
- Dnes mirror: \`bazina:/mnt/ssd/repos/dnes.git\`
|
|
- Now mirror: \`bazina:/mnt/ssd/repos/now.git\`
|
|
EOF
|
|
|
|
# Symlink to dnes logs directory
|
|
ln -sfn "$AUDIT_FILE" "$LOGS_DIR/fleet_audit_${TODAY}.md"
|
|
|
|
# --- 5. REPOSITORY COLLECTION & PUSH ---
|
|
# Copy latest human journals into archive
|
|
[ -f "$DNES_DIR/journal.md" ] && cp -f "$DNES_DIR/journal.md" "$ARCHIVE_DIR/dnes/journal.md"
|
|
[ -f "$DNES_DIR/plan.md" ] && cp -f "$DNES_DIR/plan.md" "$ARCHIVE_DIR/dnes/plan.md"
|
|
[ -f "$HOME/src/now/log" ] && cp -f "$HOME/src/now/log" "$ARCHIVE_DIR/now/log"
|
|
|
|
cd "$ARCHIVE_DIR"
|
|
git add .
|
|
if ! git diff-index --quiet HEAD -- 2>/dev/null; then
|
|
git commit -m "fleet audit ($TODAY): boxC kof17: $BOXC_KOF17_AVAIL, syncthing: $BOXC_SYNCTHING, alerts: $ALERT_COUNT"
|
|
git push origin main || echo "Warning: Push to bazina failed"
|
|
echo "Fleet journal audit committed and pushed to bazina big disk."
|
|
else
|
|
echo "Working tree clean. Nothing new to commit."
|
|
fi
|
|
|
|
# Log via now
|
|
if command -v now >/dev/null 2>&1; then
|
|
now "[FLEET-AUDIT] Daily check ($TODAY) finished: alerts=$ALERT_COUNT, boxC kof17=$BOXC_KOF17_AVAIL"
|
|
fi
|
|
|
|
echo "=== Fleet Daily Audit Complete ==="
|
|
if [ "$ALERT_COUNT" -gt 0 ]; then
|
|
echo "WARNING: $ALERT_COUNT alerts detected!"
|
|
exit 1
|
|
fi
|
|
exit 0
|