Solution — System Health Monitor

Full reference implementation. Study this after attempting the project yourself.

health_check.conf

# health_check.conf — Configuration for health_check.sh

# Thresholds (percentages)
THRESHOLD_DISK=85
THRESHOLD_MEM=80
THRESHOLD_CPU=4.0

# Logging
LOGFILE="${HOME}/projects/health-monitor/logs/health_check.log"
LOG_MAX_LINES=1000

# Optional Slack webhook
# SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

# Optional process monitoring
# WATCH_PROCS="nginx mysql"
# THRESHOLD_PROC_MEM=25

# Optional URL monitoring
# WATCH_URLS="http://localhost:8080/health"

health_check.sh

#!/usr/bin/env bash
# =============================================================================
# health_check.sh — System health monitor
#
# Usage:
#   health_check.sh             # run check, use config file
#   health_check.sh --report    # print a one-line status summary
#   health_check.sh --help      # show this help
#
# Cron example (every 5 minutes):
#   */5 * * * * /home/user/projects/health-monitor/health_check.sh
# =============================================================================
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG="${SCRIPT_DIR}/health_check.conf"

# ── defaults (overridden by config) ──────────────────────────────────────────
THRESHOLD_DISK=85
THRESHOLD_MEM=80
THRESHOLD_CPU=4.0
LOGFILE="/tmp/health_check.log"
LOG_MAX_LINES=1000

# ── load config ──────────────────────────────────────────────────────────────
[[ -f "$CONFIG" ]] && source "$CONFIG"

# ── logging ──────────────────────────────────────────────────────────────────
_log() {
    local level="$1"; shift
    local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
    echo "$msg" | tee -a "$LOGFILE"
}
log_info()  { _log "INFO " "$@"; }
log_warn()  { _log "WARN " "$@"; }
log_alert() { _log "ALERT" "$@"; }

# ── metric collectors ─────────────────────────────────────────────────────────
get_disk_pct() {
    df / | awk 'NR==2 { gsub(/%/,"",$5); print $5 }'
}

get_mem_pct() {
    free | awk '/^Mem:/ { printf "%.0f", $3/$2*100 }'
}

get_cpu_load() {
    uptime | awk -F'load average:' '{ print $2 }' | awk '{ print $1 }' | tr -d ','
}

get_disk_free_gb() {
    df -BG / | awk 'NR==2 { gsub(/G/,"",$4); print $4 }'
}

# ── threshold check ───────────────────────────────────────────────────────────
check_metric() {
    local name="$1" value="$2" threshold="$3" unit="${4:-%}"
    local breached
    breached=$(awk -v v="$value" -v t="$threshold" 'BEGIN { print (v+0 > t+0) ? "yes" : "no" }')
    if [[ "$breached" == "yes" ]]; then
        log_alert "$name at ${value}${unit} — threshold ${threshold}${unit} exceeded"
        return 1
    fi
    log_info "$name OK: ${value}${unit} (threshold: ${threshold}${unit})"
    return 0
}

# ── alerting ──────────────────────────────────────────────────────────────────
send_alert() {
    local message="$1"
    [[ -n "${SLACK_WEBHOOK_URL:-}" ]] || return 0
    curl -sf -X POST "$SLACK_WEBHOOK_URL" \
         -H "Content-Type: application/json" \
         -d "{\"text\": \"[$(hostname)] :warning: $message\"}" \
         >/dev/null && log_info "Slack alert sent" || log_warn "Slack alert failed"
}

# ── log rotation ──────────────────────────────────────────────────────────────
rotate_log() {
    [[ -f "$LOGFILE" ]] || return 0
    local lines
    lines=$(wc -l < "$LOGFILE")
    if (( lines > LOG_MAX_LINES )); then
        tail -n "$LOG_MAX_LINES" "$LOGFILE" > "${LOGFILE}.tmp"
        mv "${LOGFILE}.tmp" "$LOGFILE"
        log_info "Log rotated (was $lines lines, kept $LOG_MAX_LINES)"
    fi
}

# ── report mode ───────────────────────────────────────────────────────────────
usage() {
    echo "Usage: $0 [--report|--help]"
}

print_report() {
    printf "%-10s %-8s %-10s %-8s\n" "HOST" "DISK%" "MEM%" "CPU_LOAD"
    printf "%-10s %-8s %-10s %-8s\n" \
        "$(hostname | cut -c1-10)" \
        "$(get_disk_pct)%" \
        "$(get_mem_pct)%" \
        "$(get_cpu_load)"
}

# ── main ──────────────────────────────────────────────────────────────────────
main() {
    case "${1:-}" in
        --help|-h) usage; exit 0 ;;
        --report)  print_report; exit 0 ;;
    esac

    mkdir -p "$(dirname "$LOGFILE")"
    local alerts=0

    log_info "=== Health check started on $(hostname) ==="

    local disk_pct mem_pct cpu_load
    disk_pct=$(get_disk_pct)
    mem_pct=$(get_mem_pct)
    cpu_load=$(get_cpu_load)

    check_metric "Disk /" "$disk_pct" "$THRESHOLD_DISK" "%" || {
        (( alerts++ )) || true
        send_alert "Disk / at ${disk_pct}% (threshold: ${THRESHOLD_DISK}%)"
    }
    check_metric "Memory" "$mem_pct" "$THRESHOLD_MEM" "%" || {
        (( alerts++ )) || true
        send_alert "Memory at ${mem_pct}% (threshold: ${THRESHOLD_MEM}%)"
    }
    check_metric "CPU load" "$cpu_load" "$THRESHOLD_CPU" "" || {
        (( alerts++ )) || true
        send_alert "CPU load $cpu_load (threshold: ${THRESHOLD_CPU})"
    }

    if (( alerts == 0 )); then
        log_info "=== All systems normal — disk free: $(get_disk_free_gb)GB ==="
    else
        log_warn "=== $alerts alert(s) triggered ==="
    fi

    rotate_log
}

[[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@"

Sample Output

[2024-01-15 09:30:00] [INFO ] === Health check started on web-server-01 === [2024-01-15 09:30:00] [INFO ] Disk / OK: 45% (threshold: 85%) [2024-01-15 09:30:00] [INFO ] Memory OK: 62% (threshold: 80%) [2024-01-15 09:30:00] [INFO ] CPU load OK: 0.42 (threshold: 4.0) [2024-01-15 09:30:00] [INFO ] === All systems normal — disk free: 110GB ===

[2024-01-15 09:35:00] [INFO ] === Health check started on web-server-01 === [2024-01-15 09:35:00] [ALERT] Disk / at 91% — threshold 85% exceeded [2024-01-15 09:35:00] [INFO ] Slack alert sent [2024-01-15 09:35:00] [INFO ] Memory OK: 62% (threshold: 80%) [2024-01-15 09:35:00] [INFO ] CPU load OK: 0.42 (threshold: 4.0) [2024-01-15 09:35:00] [WARN ] === 1 alert(s) triggered ===

Key Design Decisions

Decision	Why
awk for float comparison	Bash (( )) only handles integers
tee -a in _log()	Log to file AND show on terminal simultaneously
[[ "${BASH_SOURCE[0]}" == "$0" ]] guard	Allows sourcing for unit testing
|| true on (( alerts++ ))	Prevents -e from exiting when alerts was 0
Config sourced with source	Simple key=value config, no parsing needed

---

enhancements | overview