#!/bin/bash
#
# Checkmk local check for bashclub-zsync: one service "zsync <config>" per configuration
# Installed by the package as /usr/lib/check_mk_agent/local/bashclub_zsync (link), if the agent is installed.
#
# Reads only the status files written by bashclub-zsync and /proc, never calls zfs (it could hang the same way).
# The status files are written by the runs, also by unprivileged users: they are data, never code. The check
# reads settings from the configuration as text (never runs it), accepts only a configuration file that
# belongs to root and that no one else may write, and uses only numbers that are numbers.
# One folder per user that runs bashclub-zsync (only folders whose owner has the name of the folder count); the
# status of a configuration is taken from the folder of its run_as user, otherwise root, otherwise the user with
# the newest result (e.g. a cron job of another user):
#   /run/bashclub-zsync/<user>/<config>.state       active run: pid, phase, dataset
#   /var/lib/bashclub-zsync/<user>/<config>.last    result of the last run and of the last successful run
#   /var/lib/bashclub-zsync/<user>/<config>.history start times of the last runs
#   /var/lib/bashclub-zsync/<user>/<config>.skipped times of runs skipped because of an active run
#   /var/lib/bashclub-zsync/<user>/<config>.datasets state per dataset: dataset, state, since, detail
#   /var/lib/bashclub-zsync/root/<config>.check     state of this check (transfer progress, blocked processes),
#                                                   written only into this folder of root
#
# Checks:
#   age of the last successful run   not for removable disks (trigger=autorun or zsync-autorun@) without monitor_max_age;
#                                    interval = from the systemd timer bashclub-zsync@<name>.timer, otherwise the
#                                    median of the start intervals (or monitor_max_age in the config),
#                                    WARN after interval + max(20 %, 5 min) + duration of the last successful run
#                                    (or of the active run, if it takes longer),
#                                    CRIT after 3 intervals + duration; learning phase until 5 intervals: at most WARN
#   last run failed                  WARN
#   phase without transfer too long  listing, snapshots, deletion, checkzfs
#   transfer without progress        bytes read by "zfs receive" do not change
#   blocked processes                processes of the run in state D (uninterruptible, e.g. waiting for zfs)
#   aborted run                      status file of a run whose process does not exist anymore
#   skipped runs                     runs skipped because the previous run is still active
#   state per dataset                CRIT: no common snapshot with the source or target without snapshots (full
#                                    replication needed); WARN: base will be rotated out (chain at risk), newest target
#                                    snapshot missing on the source (diverged), replica without source or not selected
#                                    anymore (after orphan_grace, acknowledged with bashclub:zsync-orphan=keep)

RUN_DIR=${ZSYNC_RUN_DIR:-/run/bashclub-zsync}
LIB_DIR=${ZSYNC_LIB_DIR:-/var/lib/bashclub-zsync}
CHECK_DIR=${ZSYNC_CHECK_DIR:-$LIB_DIR/root}
PROC=${ZSYNC_PROC:-/proc}
SYSTEMCTL=${ZSYNC_SYSTEMCTL:-systemctl}
CONF_DIR=${ZSYNC_CONF_DIR:-/etc/bashclub}
DEFAULT_DIR=${ZSYNC_DEFAULT_DIR:-/etc/default}
CONF_OWNER=${ZSYNC_CONF_OWNER:-0}
NOW=${ZSYNC_NOW:-$(date +%s)}

# thresholds in seconds, can be changed in /etc/bashclub/zsync-checkmk.conf
phase_warn=900          # phases without transfer: 15 min
phase_crit=3600         #                          1 h
stall_warn=600          # transfer without progress: 10 min
stall_crit=1800         #                            30 min
blocked_warn=300        # process in state D: 5 min
blocked_crit=900        #                     15 min
skipped_warn=3          # skipped runs during one active run
learn_intervals=5       # start intervals needed before the age check can become critical
orphan_grace=86400      # replicas without source / not selected anymore: WARN after 1 day (renaming, migration)
CONF_FILE=${ZSYNC_CHECK_CONF:-/etc/bashclub/zsync-checkmk.conf}
if [ -r "$CONF_FILE" ]; then . "$CONF_FILE"; fi

# 3725 -> 1h 2m
function duration() {
    local s=$1
    if [ $s -lt 0 ]; then s=0; fi
    if [ $s -ge 86400 ]; then echo "$((s / 86400))d $((s % 86400 / 3600))h"
    elif [ $s -ge 3600 ]; then echo "$((s / 3600))h $((s % 3600 / 60))m"
    elif [ $s -ge 60 ]; then echo "$((s / 60))m"
    else echo "${s}s"; fi
}

# "a, b, c, d" -> "a, b, c (+1 more)"
function shortlist() {
    local -a items=("$@")
    local text
    text=$(printf '%s, ' "${items[@]:0:3}")
    text=${text%, }
    if [ ${#items[@]} -gt 3 ]; then text+=" (+$((${#items[@]} - 3)) more)"; fi
    echo "$text"
}

# 26h, 90m, 2d or seconds -> seconds
function seconds() {
    case $1 in
        *d) echo $(( ${1%d} * 86400 )) ;;
        *h) echo $(( ${1%h} * 3600 )) ;;
        *m) echo $(( ${1%m} * 60 )) ;;
        *s) echo $(( ${1%s} )) ;;
        *)  echo $(( $1 )) ;;
    esac
}

# key=value file into the associative array named $2; keys with numbers (times, pids, counts) keep their value
# only if it is a number (bash would evaluate other content in arithmetic expressions)
function read_file() {
    local -n arr=$2
    local key value
    arr=()
    if [[ -f $1 ]]; then
        while IFS='=' read -r key value; do
            if [[ -z $key ]]; then continue; fi
            if [[ $key =~ ^(config|phase|dataset|progress_key)$ ]] || [[ $value =~ ^[0-9]+$ ]]; then arr[$key]=$value; fi
        done < "$1"
    fi
}

# number or empty
function number() {
    if [[ $1 =~ ^[0-9]+$ ]]; then echo "$1"; fi
}

# configuration of a status file: a regular file *.conf of root, not writable for group and others
function trusted_conf() {
    [[ $1 == /*.conf ]] && [[ -f $1 ]] && [[ ! -L $1 ]] || return 1
    [[ $(stat -c '%u' "$1" 2>/dev/null) == "$CONF_OWNER" ]] || return 1
    [[ $(( 8#$(stat -c '%a' "$1") & 8#022 )) -eq 0 ]]
}

# value of a setting without running the configuration (last line key=..., quotes removed)
function conf_value() {
    local line value
    line=$(grep -E "^[[:space:]]*$2=" "$1" 2>/dev/null | tail -1)
    [[ -n $line ]] || return 0
    value=${line#*=}
    if [[ $value == \"* ]]; then value=${value#\"}; value=${value%%\"*}
    elif [[ $value == \'* ]]; then value=${value#\'}; value=${value%%\'*}
    else value=${value%%[[:space:]]*}; fi
    printf '%s\n' "$value"
}

# fields of /proc/<pid>/stat after "pid (comm) ": state ppid ... (starttime is field 20)
function stat_fields() {
    sed 's/.*) //' "$PROC/$1/stat" 2>/dev/null
}

# pid and all its descendants (cat continues if a process ends in the meantime)
function process_tree() {
    local -A children=()
    local pid ppid queue
    while read -r pid ppid; do
        children[$ppid]+=" $pid"
    done < <(cat "$PROC"/[0-9]*/stat 2>/dev/null | sed -nE 's/^([0-9]+) \(.*\) . ([0-9]+) .*/\1 \2/p')
    queue=$1
    while [[ -n $queue ]]; do
        pid=${queue%% *}; queue=${queue#"$pid"}; queue=${queue# }
        echo $pid
        queue="$queue${children[$pid]}"; queue=${queue# }
    done
}

state_level=0
state_text=()
function raise() {
    if [ $1 -gt $state_level ]; then state_level=$1; fi
    state_text+=("$2")
}
function level_mark() { case $1 in 1) echo " (!)" ;; 2) echo " (!!)" ;; esac; }

# names of the user folders below $1 whose owner has the name of the folder
function user_dirs() {
    local d
    for d in "$1"/*/; do
        d=${d%/}
        if [[ -d $d ]] && [[ ! -L $d ]] && [[ $(stat -c %U "$d" 2>/dev/null) == "${d##*/}" ]]; then echo "${d##*/}"; fi
    done
}

users=$( { user_dirs "$RUN_DIR"; user_dirs "$LIB_DIR"; } | sort -u | paste -sd' ')
ids=$(for u in $users; do ls "$RUN_DIR/$u" "$LIB_DIR/$u" 2>/dev/null; done | sed -nE 's/\.(state|last|history)$//p' | sort -u)
for id in $ids; do
    declare -A st=() last=() chk=()
    # configuration: from a status file of any user, it has to match the name of the status file
    conf=
    for u in $users; do
        for f in "$RUN_DIR/$u/$id.state" "$LIB_DIR/$u/$id.last"; do
            c=$(sed -n 's/^config=//p' "$f" 2>/dev/null | head -1)
            if [[ -n $c ]] && [[ $(tr '/' '_' <<< "$c") == "$id" ]]; then conf=$c; break 2; fi
        done
    done
    # configuration removed: no service anymore; a configuration that is not trusted is not read
    if [[ -z $conf ]] || [[ ! -f $conf ]]; then continue; fi
    if ! trusted_conf "$conf"; then
        echo "1 \"zsync $(basename "$conf" .conf)\" - configuration $conf is not a file of root that only root may write, not checked"
        continue
    fi
    # user that runs the configuration: run_as, otherwise root, otherwise the user with the newest result
    owner=$(conf_value "$conf" run_as)
    owner=${owner:-root}
    if [[ ! -f $LIB_DIR/$owner/$id.last ]] && [[ ! -f $RUN_DIR/$owner/$id.state ]] && [[ -z $(conf_value "$conf" run_as) ]]; then
        newest=0
        for u in $users; do
            end=$(number "$(sed -n 's/^end=//p' "$LIB_DIR/$u/$id.last" 2>/dev/null)")
            if [[ -f $RUN_DIR/$u/$id.state ]] || [ ${end:-0} -gt $newest ]; then owner=$u; newest=${end:-0}; fi
        done
    fi
    if ! [[ " $users " == *" $owner "* ]]; then continue; fi
    run_d=$RUN_DIR/$owner
    lib_d=$LIB_DIR/$owner
    read_file "$run_d/$id.state" st
    read_file "$lib_d/$id.last" last
    read_file "$CHECK_DIR/$id.check" chk
    name=$(basename "$conf" .conf)
    state_level=0
    state_text=()
    metrics=()
    new_chk=()

    #### active run
    running=0
    if [[ -n ${st[pid]} ]]; then
        fields=$(stat_fields ${st[pid]})
        if [[ -n $fields ]] && [[ $(cut -d' ' -f20 <<< "$fields") == "${st[pid_start]}" ]]; then
            running=1
        elif [[ ${last[start]} != "${st[run_start]}" ]]; then
            # the run did not write its result (.last of an earlier run): killed, crashed or power loss
            raise 1 "run started $(duration $((NOW - ${st[run_start]:-NOW}))) ago was aborted in phase ${st[phase]}${st[dataset]:+ (${st[dataset]})}$(level_mark 1)"
        fi
    fi

    if [ $running -eq 1 ]; then
        phase=${st[phase]}
        phase_age=$((NOW - ${st[phase_since]:-NOW}))
        state_text+=("running for $(duration $((NOW - ${st[run_start]}))), phase $phase${st[dataset]:+ ${st[dataset]}} since $(duration $phase_age)")
        tree=$(process_tree ${st[pid]})

        # processes in state D: time since first seen by this check
        for pid in $tree; do
            fields=$(stat_fields $pid)
            if [[ ${fields%% *} == D ]]; then
                key="blocked_${pid}_$(cut -d' ' -f20 <<< "$fields")"
                since=$(number "${chk[$key]}"); since=${since:-$NOW}
                new_chk+=("$key=$since")
                blocked=$((NOW - since))
                cmd=$(tr '\0' ' ' < "$PROC/$pid/cmdline" 2>/dev/null | cut -c1-80)
                wchan=$(cat "$PROC/$pid/wchan" 2>/dev/null)
                if [ $blocked -ge $blocked_crit ]; then level=2; elif [ $blocked -ge $blocked_warn ]; then level=1; else level=0; fi
                if [ $level -gt 0 ]; then
                    raise $level "process $pid (${cmd% }) blocked for $(duration $blocked) in ${wchan:-?}$(level_mark $level)"
                fi
            fi
        done

        if [[ $phase == send ]]; then
            # progress: bytes read by all "zfs receive" of the run
            bytes=0
            for pid in $tree; do
                if tr '\0' ' ' < "$PROC/$pid/cmdline" 2>/dev/null | grep -qE '(^|[ /])zfs receive '; then
                    read_bytes=$(sed -n 's/^rchar: //p' "$PROC/$pid/io" 2>/dev/null)
                    bytes=$((bytes + ${read_bytes:-0}))
                fi
            done
            # the transfer is identified by dataset and start of the phase
            progress_key="${st[phase_since]}_${st[dataset]}"
            if [[ ${chk[progress_key]} != "$progress_key" ]]; then
                # first check of this transfer: nothing read yet counts from the start of the phase
                if [ $bytes -eq 0 ]; then progress_since=${st[phase_since]:-$NOW}; else progress_since=$NOW; fi
            elif [[ ${chk[progress_bytes]} != "$bytes" ]]; then
                progress_since=$NOW
            else
                progress_since=${chk[progress_since]}
            fi
            new_chk+=("progress_key=$progress_key" "progress_bytes=$bytes" "progress_since=$progress_since")
            stalled=$((NOW - progress_since))
            if [ $stalled -ge $stall_crit ]; then level=2; elif [ $stalled -ge $stall_warn ]; then level=1; else level=0; fi
            if [ $level -gt 0 ]; then raise $level "no transfer progress for $(duration $stalled)$(level_mark $level)"; fi
        else
            if [ $phase_age -ge $phase_crit ]; then level=2; elif [ $phase_age -ge $phase_warn ]; then level=1; else level=0; fi
            if [ $level -gt 0 ]; then raise $level "phase $phase takes $(duration $phase_age)$(level_mark $level)"; fi
        fi

        # runs skipped while this run is active
        skipped=$(awk -v s=${st[run_start]} '$1 >= s' "$lib_d/$id.skipped" 2>/dev/null | grep -c .)
        if [ $skipped -ge $skipped_warn ]; then
            raise 1 "$skipped runs skipped while this run is active$(level_mark 1)"
        elif [ $skipped -gt 0 ]; then
            state_text+=("$skipped runs skipped while this run is active")
        fi
    fi
    if [[ -d $CHECK_DIR ]] && [[ ! -L $CHECK_DIR ]] && [[ -O $CHECK_DIR ]]; then
        printf '%s\n' "${new_chk[@]}" > "$CHECK_DIR/$id.check.tmp" 2>/dev/null && mv -f "$CHECK_DIR/$id.check.tmp" "$CHECK_DIR/$id.check" 2>/dev/null
    fi

    #### last run
    if [[ -n ${last[rc]} ]]; then
        metrics+=("last_duration=${last[duration]}" "datasets_failed=${last[datasets_failed]:-0}")
        if [ ${last[rc]} -ne 0 ]; then
            raise 1 "last run $(duration $((NOW - ${last[end]}))) ago failed (exit code ${last[rc]}, ${last[datasets_failed]:-0} of ${last[datasets]:-0} datasets failed)$(level_mark 1)"
        fi
    fi

    #### state per dataset (written by the last run)
    declare -a ds_crit=() ds_risk=() ds_div=() ds_orphan=() ds_failed=() ds_blocked=()
    kept=0; nosnap=0; orphan_new=0; rolled_back=()
    if [[ -f $lib_d/$id.datasets ]]; then
        while IFS=$'\t' read -r ds dstate since detail; do
            since=$(number "$since")
            age_text=$(duration $((NOW - ${since:-NOW})))
            case $dstate in
                broken|empty) ds_crit+=("$ds ($detail, since $age_text)") ;;
                at_risk) ds_risk+=("$ds ($detail)") ;;
                cleanup_blocked) ds_blocked+=("$ds ($detail, since $age_text)") ;;
                diverged) ds_div+=("$ds ($detail, since $age_text)") ;;
                failed) ds_failed+=("$ds (since $age_text)") ;;
                orphan_missing|orphan_unselected)
                    if [ $((NOW - ${since:-NOW})) -ge $orphan_grace ]; then
                        ds_orphan+=("$ds (${detail%% (*}, since $age_text)")
                    else
                        ((orphan_new++))
                    fi ;;
                orphan_kept) ((kept++)) ;;
                rolled_back) rolled_back+=("$ds ($detail)") ;;
                no_snapshots) ((nosnap++)) ;;
            esac
        done < "$lib_d/$id.datasets"
    fi
    if [ ${#ds_crit[@]} -gt 0 ]; then
        raise 2 "full replication needed for ${#ds_crit[@]} datasets: $(shortlist "${ds_crit[@]}")$(level_mark 2)"
    fi
    if [ ${#ds_risk[@]} -gt 0 ]; then
        raise 1 "chain at risk, the next rotation on the source breaks it: $(shortlist "${ds_risk[@]}")$(level_mark 1)"
    fi
    if [ ${#ds_div[@]} -gt 0 ]; then
        raise 1 "newest target snapshot missing on the source: $(shortlist "${ds_div[@]}")$(level_mark 1)"
    fi
    if [ ${#ds_blocked[@]} -gt 0 ]; then
        raise 1 "rotation on the target blocked: $(shortlist "${ds_blocked[@]}")$(level_mark 1)"
    fi
    if [ ${#ds_orphan[@]} -gt 0 ]; then
        raise 1 "${#ds_orphan[@]} datasets only on the target: $(shortlist "${ds_orphan[@]}")$(level_mark 1)"
    fi
    if [ ${#ds_failed[@]} -gt 0 ]; then state_text+=("failed: $(shortlist "${ds_failed[@]}")"); fi
    if [ $orphan_new -gt 0 ]; then state_text+=("$orphan_new new datasets only on the target"); fi
    if [ $kept -gt 0 ]; then state_text+=("$kept kept replicas without source"); fi
    if [ ${#rolled_back[@]} -gt 0 ]; then state_text+=("rolled back by the last run: $(shortlist "${rolled_back[@]}")"); fi
    if [ $nosnap -gt 0 ]; then state_text+=("$nosnap datasets without matching snapshots"); fi

    #### age of the last successful run
    max_age=$(conf_value "$conf" monitor_max_age)
    if [[ -n $max_age ]] && ! [[ $max_age =~ ^[0-9]+[dhms]?$ ]]; then
        state_text+=("invalid monitor_max_age=$max_age ignored")
        max_age=
    fi
    # removable disk (trigger=autorun or used by zsync-autorun@): runs only when the disk is connected
    autorun=$(conf_value "$conf" trigger)
    if [[ $autorun != "autorun" ]]; then
        for link in "$DEFAULT_DIR"/zsync-autorun@*; do
            if [[ -e $link ]] && [[ $(readlink -f "$link") == "$(readlink -f "$conf")" ]]; then autorun=autorun; fi
        done
    fi
    if [[ $max_age == 0 ]]; then
        state_text+=("age check disabled (monitor_max_age=0)")
    elif [[ -z $max_age ]] && [[ $autorun == "autorun" ]]; then
        state_text+=("no age check for a removable disk (autorun), set monitor_max_age for one")
    elif [[ -z ${last[ok_end]} ]]; then
        state_text+=("no successful run yet")
    else
        age=$((NOW - ${last[ok_end]}))
        ok_duration=${last[ok_duration]:-0}
        # an active run that takes longer than the last one is watched by the phase, progress and D state checks
        if [ $running -eq 1 ] && [ $((NOW - ${st[run_start]})) -gt $ok_duration ]; then
            ok_duration=$((NOW - ${st[run_start]}))
        fi
        learning=0
        # systemd timer of this configuration (bashclub-zsync@<name>.timer): interval from its last and next start
        # (last and next instead of the schedule: a schedule like weekdays 6-22 every 15 min has a pause at night)
        timer_interval=
        if [[ $(dirname "$conf") == "$CONF_DIR" ]] && command -v "$SYSTEMCTL" > /dev/null \
            && "$SYSTEMCTL" is-active --quiet "bashclub-zsync@$name.timer" 2>/dev/null; then
            timer_last=$("$SYSTEMCTL" show -p LastTriggerUSec --value "bashclub-zsync@$name.timer" 2>/dev/null)
            timer_next=$("$SYSTEMCTL" show -p NextElapseUSecRealtime --value "bashclub-zsync@$name.timer" 2>/dev/null)
            if [[ -n $timer_last && $timer_last != n/a ]]; then timer_last=$(date -d "$timer_last" +%s 2>/dev/null); else timer_last=; fi
            if [[ -n $timer_next && $timer_next != n/a ]]; then timer_next=$(date -d "$timer_next" +%s 2>/dev/null); else timer_next=; fi
            # timer not triggered yet (e.g. started by hand): the last successful run is the last start
            if [[ -z $timer_last ]]; then timer_last=${last[ok_start]}; fi
            if [[ -n $timer_last && -n $timer_next ]] && [ $timer_next -gt $timer_last ]; then
                timer_interval=$((timer_next - timer_last))
            fi
        fi
        if [[ -n $max_age ]]; then
            warn_at=$(seconds "$max_age")
            crit_at=$((warn_at * 3))
            source_text="monitor_max_age=$max_age"
        elif [[ -n $timer_interval ]]; then
            interval=$timer_interval
            grace=$((interval / 5))
            if [ $grace -lt 300 ]; then grace=300; fi
            warn_at=$((interval + grace + ok_duration))
            crit_at=$((3 * interval + ok_duration))
            if [ $crit_at -le $warn_at ]; then crit_at=$((warn_at + grace)); fi
            source_text="interval $(duration $interval) from bashclub-zsync@$name.timer"
        else
            # median of the intervals between the start times
            intervals=$(sort -n "$lib_d/$id.history" 2>/dev/null | awk 'NR > 1 && $1 > p { print $1 - p } { p = $1 }' | sort -n)
            count=$(grep -c . <<< "$intervals")
            if [ $count -eq 0 ]; then
                warn_at=
                source_text="learning 0/$learn_intervals"
            else
                interval=$(awk '{ a[NR] = $1 } END { print (NR % 2) ? a[(NR + 1) / 2] : int((a[NR / 2] + a[NR / 2 + 1]) / 2) }' <<< "$intervals")
                grace=$((interval / 5))
                if [ $grace -lt 300 ]; then grace=300; fi
                warn_at=$((interval + grace + ok_duration))
                crit_at=$((3 * interval + ok_duration))
                if [ $crit_at -le $warn_at ]; then crit_at=$((warn_at + grace)); fi
                if [ $count -lt $learn_intervals ]; then
                    learning=1
                    source_text="learning $count/$learn_intervals, preliminary interval $(duration $interval)"
                else
                    source_text="interval $(duration $interval) from $count runs"
                fi
            fi
        fi
        text="last successful run $(duration $age) ago ($source_text)"
        if [[ -n $warn_at ]]; then
            metrics=("last_success_age=$age;$warn_at;$crit_at" "${metrics[@]}")
            if [ $age -ge $crit_at ] && [ $learning -eq 0 ]; then level=2
            elif [ $age -ge $warn_at ]; then level=1
            else level=0; fi
            if [ $level -gt 0 ]; then raise $level "$text$(level_mark $level)"; else state_text+=("$text"); fi
        else
            metrics=("last_success_age=$age" "${metrics[@]}")
            state_text+=("$text")
        fi
    fi

    text=$(printf '%s, ' "${state_text[@]}")
    text=${text%, }
    perf=$(IFS='|'; echo "${metrics[*]}")
    echo "$state_level \"zsync $name\" ${perf:--} ${text:-no data}"
done
