#!/bin/bash
#
# bashclub zfs replication script
# Author: (C) 2024 Thorsten Spille <thorsten@spille-edv.de>

# version is taken from the installed package (set by the release tag)
__version__=v$(dpkg-query -W -f='${Version}' bashclub-zsync 2>/dev/null || echo dev)

# also search next to this script (manual installation, e.g. on a TrueNAS dataset)
PATH="$(dirname "$(readlink -f "$0")"):/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"

prog="$(basename $0)"
zfs=$(which zfs)
ssh=$(which ssh)
scp=$(which scp)
grep=$(which grep)
uniq=$(which uniq)
cut=$(which cut)
tail=$(which tail)
wc=$(which wc)
tr=$(which tr)
sed=$(which sed)
zfs_auto_snapshot=$(which zfs-auto-snapshot)
checkzfs=$(which checkzfs)
mv=$(which mv)
rsm=$(which remote-snapshot-manager)

rc=0
debug=

#### default config file, can be changed with parameter -c
conf=/etc/bashclub/zsync.conf

#### default values in config file
# replication target path on local machine 
target=pool/dataset

# ssh address of remote machine
source=user@host

# ssh port of remote machine
sshport=22

# zfs user parameter to identify filesystems/volumes to replicate
tag=bashclub:zsync

# pipe separated list of snapshot name filters
snapshot_filter="hourly|daily|weekly|monthly"

# minimum count of snapshots per filter to keep
min_keep=3

# number of zfs snapshots to keep on source (0 or 1 = snapshot function disabled)
zfs_auto_snapshot_keep=0

# make snapshot via zfs-auto-snapshot before replication
zfs_auto_snapshot_label="backup"

# select snapshot engine: "zas" or "internal"
# keep "zas" here for existing configs without this setting; new configs get "internal"
zfs_auto_snapshot_engine="zas"

# disable checkzfs with value > 0
checkzfs_disabled=0

# do not connect to source host (if replication is local)
checkzfs_local=0

# set checkzfs parameter "--prefix"
checkzfs_prefix=zsync

# set checkzfs maximum age of last snapshot in minutes (comma separated => warn,crit)
checkzfs_max_age=1500,6000

# set checkzfs maximum count of snapshots per dataset (comma separated => warn,crit)
checkzfs_max_snapshot_count=150,165

# set where to move the checkzfs output (0 = local machine, 1 = source machine)
checkzfs_spool=0

# set maxmimum age of checkzfs data in seconds (hourly = 3900, daily = 87000)
checkzfs_spool_maxage=87000

usage() {
	cat >&2 << EOF
$prog Version $__version__
-------------------------------------------------------------------------------
usage: $prog [-h] [-d] [-c CONFIG]
    creates a mirrored replication of configured zfs filesystems / volumes
    -c CONFIG    Configuration file for this script
    -d           Debug mode
-------------------------------------------------------------------------------
    (C) 2024     by Spille IT Solutions for bashclub (github.com/bashclub)
                 Author: Thorsten Spille <thorsten@spille-edv.de>
-------------------------------------------------------------------------------
EOF
	exit $1
}

while getopts "hdc:" opt; do
  case $opt in
    h) usage 0 ;;
    c) conf=$OPTARG ;;
    d) debug=-v ;;
    *) usage 1 ;;
  esac
done
shift $((OPTIND-1))

function log() {
    echo -e "$(date +'%b %d %T') $1"
}

if which checkzfs; then
    if [ $checkzfs_disabled -eq 0 ]; then
        checkzfs=$(which checkzfs)
        log "[INFO] installed checkzfs found: $checkzfs"
    else
        log "[INFO] checkzfs is disabled by configuration."
    fi
else
    checkzfs_disabled=1
    log "[WARN] checkzfs is disabled. for monitoring please install:\n\thttps://code.bashclub.org/bashclub/checkzfs\n"
fi

# load config file
if [ -f $conf ]; then
    log "[INFO] Reading configuration $conf"
    source $conf
else
    mkdir -p $(dirname $conf)
    cat << EOF > $conf
target=$target
source=$source
sshport=$sshport
tag=$tag
snapshot_filter="$snapshot_filter"
min_keep=$min_keep
zfs_auto_snapshot_keep=$zfs_auto_snapshot_keep
zfs_auto_snapshot_label=$zfs_auto_snapshot_label
zfs_auto_snapshot_engine=internal
checkzfs_disabled=$checkzfs_disabled
checkzfs_local=$checkzfs_local
checkzfs_prefix=$checkzfs_prefix
checkzfs_max_age=$checkzfs_max_age
checkzfs_max_snapshot_count=$checkzfs_max_snapshot_count
checkzfs_spool=$checkzfs_spool
checkzfs_spool_maxage=$checkzfs_spool_maxage
EOF
    log "[INFO] Initial config file created. Please adjust and restart script. Exiting..."
    usage 0
fi

if [[ $source == "" ]]; then
    log "[INFO] source is empty, switching to local mode."
    ssh=
    zsync_sshport=
    log "[INFO] Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\ŧzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tzfs_auto_snapshot_engine=$zfs_auto_snapshot_engine\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
else
    zsync_sshport=-p$sshport
    log "[INFO] Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\tzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tzfs_auto_snapshot_engine=$zfs_auto_snapshot_engine\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
fi

controlmaster=-oControlMaster=auto
controlpath=-oControlPath=~/.ssh/bashclub-zsync-%r@%h-%p
controlpersist=-oControlPersist=15

local_os_id=$($grep -E "^ID=" /etc/os-release | $cut -d'=' -f2)
remote_os_id=$($ssh $controlmaster $controlpath $controlpersist $source $zsync_sshport "grep -E \"^ID=\" /etc/os-release | cut -d'=' -f2")

if [[ $local_os_id == "freebsd" ]]; then
    local_aes=$($grep -o AES /var/run/dmesg.boot | $uniq)
else
    local_aes=$($grep -m1 -o aes /proc/cpuinfo | $uniq)
fi

if [[ $remote_os_id == "freebsd" ]]; then
    remote_aes=$($ssh $controlmaster $controlpath $controlpersist $source $zsync_sshport "grep -o AES /var/run/dmesg.boot | uniq")
else
    remote_aes=$($ssh $controlmaster $controlpath $controlpersist $source $zsync_sshport "grep -m1 -o aes /proc/cpuinfo | uniq")
fi

if [[ $local_aes == "aes" ]] && [[ $remote_aes == "aes" ]]; then
    if [[ $debug == "-v" ]]; then log "[DEBUG] Switching to cipher aes256-gcm@openssh.com"; fi
    sshcipher=-caes256-gcm@openssh.com
else
    if [[ $debug == "-v" ]]; then log "[DEBUG] Using default cipher chacha20-poly1305@openssh.com"; fi
    sshcipher=-cchacha20-poly1305@openssh.com
fi

include_list="[INFO] Included datasets:\n"
exclude_list="[INFO] Excluded datasets:\n"
checkzfs_headline="[INFO] checkzfs filter:"
checkzfs_filter=
IFS=$'\n'
for zvol in $($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs get -H -o name,value,source -t filesystem,volume $tag"); do
    name=$(echo $zvol | $cut -f1)
    if [[ "$(echo $zvol | $cut -f2)" == "subvols" ]]; then
        if [[ $(echo $zvol | $grep -vE "local|received") ]]; then
            include_list="${include_list}\t${name}\n"
            syncvols=("${syncvols[@]}" "$name")
        elif [[ $(echo $zvol | $grep "local") ]]; then
            checkzfs_filter="#$name/|$checkzfs_filter"
        fi
    elif [[ "$(echo $zvol | $cut -f2)" == "all" ]]; then
        if [[ $(echo $zvol | $grep -v received) ]]; then
            include_list="${include_list}\t${name}\n"
            syncvols=("${syncvols[@]}" "$name")
        fi
        if [[ $(echo $zvol | $grep "local") ]]; then
            checkzfs_filter="#$name|$checkzfs_filter"
        fi
    else
        exclude_list="${exclude_list}\t${name}\n"
#        if [[ $(echo $zvol | $grep "local") ]]; then
#            checkzfs_filter="!#$name|$checkzfs_filter"
#        fi
    fi
done
checkzfs_filter=${checkzfs_filter::-1}
log "$include_list"
log "$exclude_list"
log "$checkzfs_headline\ncheckzfs_filter=$checkzfs_filter\n"

if ! $zfs list $target > /dev/null 2>&1 ; then
    if [[ $debug == "-v" ]]; then log "[DEBUG] $target does not exist. Creating..."; fi
    $zfs create -o canmount=noauto -o com.sun:auto-snapshot=false $target
else
    if [[ $debug == "-v" ]]; then log "[DEBUG] $target exists, check auto-snapshot...";fi
    if [[ $($zfs get -H -o value,source com.sun:auto-snapshot $target) != "false	local" ]]; then
        $zfs set com.sun:auto-snapshot=false $target
    fi
fi

if [ $zfs_auto_snapshot_keep -gt 1 ]; then
    if [[ $zfs_auto_snapshot_engine == "zas" ]]; then
        if [[ $debug == "-v" ]]; then log "[DEBUG] Running zfs-auto-snapshot"; fi
        $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //"
    fi

    if [[ $snapshot_filter == "" ]]; then
        snapshot_filter="$zfs_auto_snapshot_label"
    else
        snapshot_filter="$snapshot_filter|$zfs_auto_snapshot_label"
    fi
fi

for name in "${syncvols[@]}"; do
    log "[INFO] Replicate $name"
    fstype=$($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source zfs get -H -o value type $name)
    if [[ $fstype == "filesystem" ]]; then
        mp=-xmountpoint
        cm=-ocanmount=noauto
    else
        mp=
        cm=
    fi

    if [ $zfs_auto_snapshot_keep -gt 1 ]; then
        if [[ $zfs_auto_snapshot_engine == "internal" ]]; then
            if [[ $debug == "-v" ]]; then log "[DEBUG] Running remote-snapshot-manager"; fi
            $rsm $debug -d $name -k $zfs_auto_snapshot_keep -l $zfs_auto_snapshot_label -h $source -p $sshport
        fi
    fi

    if [[ $($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"@.*($snapshot_filter)\" | wc -l | tr -d ' '") -gt 0 ]]; then
        IFS=$' '
        if ! $zfs list -H $target/$name > /dev/null 2>&1 ; then
            if [[ $debug == "-v" ]]; then log "[DEBUG] $target/$name does not exist"; fi
            prefix=""
            for part in $(echo $target/$(echo $name | $cut -d'/' -f1) | $sed "s/\// /g"); do
                if [ $($zfs list $prefix$part > /dev/null 2>&1 ; echo $?) -gt 0 ]; then
                    if [[ $debug == "-v" ]]; then log "[DEBUG] $prefix$part does not exist"; fi
                    log "[INFO] Creating $prefix$part"
                    $zfs create -o canmount=noauto $autosnap -p $prefix$part
                fi
                prefix="$prefix$part/"
            done

            if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start initial replication"; fi
            IFS=$'\n'
            for snap in $($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"@.*($snapshot_filter)\" | tail -1"); do
                log "[INFO] Start initial replication: $snap => $target/$(echo $name | $cut -d'/' -f1)"
                $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs send -w -p $debug $snap" | $zfs receive $mp $cm -x $tag -x com.sun:auto-snapshot $debug -dF $target/$(echo $name | $cut -d'/' -f1)
                if [ $? -gt $rc ]; then rc=1; log "[ERROR] initial replication to $target/$name failed."; fi
            done
        fi

        if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start incremental replication"; fi
        guid=$($zfs list -H -o guid -s creation -t snapshot $target/$name | $tail -1)
        if [[ $guid != "" ]]; then
            last=$($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1")
            if [[ $last != "" ]]; then
                IFS=$'\n'
                if [[ $fstype == "filesystem" ]] && [[ $($zfs get -H -o value canmount $target/$name) != "noauto" ]]; then
                    $zfs set canmount=noauto $target/$name
                fi
                for snap in $($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"@.*($snapshot_filter)\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do
                    log "[INFO] Replicating delta of $last => $snap to $target/$name"
                    $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x $tag -x com.sun:auto-snapshot -F $debug $target/$name
                    if [ $? -gt $rc ]; then rc=1; log "[ERROR] incremental replication to $target/$name failed."; fi
                    last=$snap
                done
            else
                log "[ERROR] No matching snapshot (guid: $guid) found on source filesystem. This can be outdated snapshots on target or a previously deleted and new created dataset $name on the source filesystem."
                if [ $rc -eq 0 ]; then rc=1; fi
            fi
        else
            log "[ERROR] No snapshot found on $target/$name to add incremental snapshots to. The target dataset (with all children) needs to be deleted and recreated via replication."
            if [ $rc -eq 0 ]; then rc=1; fi
        fi

        if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start deletion of old snapshots"; fi
        filter=$(echo -e $snapshot_filter | sed "s/|/\n/g")
        IFS=$'\n'
        for interval in $filter ; do
            if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Checking interval $interval"; fi
            guid=$($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep -E \"@.*$interval\" | cut -f1 | tail -1")
            if [[ "$(echo -e "$guid" | sed 's/\n//g')" != "" ]]; then
                snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $grep --after-context=200 $guid | $grep -v $guid | $cut -f1)
                snap_count=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $wc -l | $tr -d ' ')

                for snap in $snaps_to_delete; do
                    if [[ $snap_count -gt $min_keep ]]; then
                        log "[INFO] Deleting $snap"
                        if [[ $debug == "-v" ]]; then log "[DEBUG] $name - snap_count=$snap_count, min_keep=$min_keep"; fi
                        $zfs destroy $debug $snap
                        snap_count=$(expr $snap_count - 1)
                    else
                        if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Skipping deletion of $snap. snap_count=$snap_count, min_keep=$min_keep"; fi
                    fi
                done
            fi
        done
    else
        if [[ $debug == "-v" ]]; then log "[DEBUG] $name - No snapshots found with filter $snapshot_filter"; fi
    fi
    log "[INFO] Replication of $name finished."
done

if [ $checkzfs_disabled -eq 0 ]; then
    log "[INFO] Running checkzfs..."
    echo "<<<local>>>" > /tmp/${checkzfs_spool_maxage}_${checkzfs_prefix}

    if [[ $checkzfs_local -eq 0 ]]; then
        c_key=--source
        c_value=$source:$sshport
    fi

    if [[ $debug == "-v" ]]; then log "[DEBUG] Command:\n$checkzfs $c_key $c_value --output checkmk --threshold $checkzfs_max_age --maxsnapshots $checkzfs_max_snapshot_count --prefix $checkzfs_prefix --replicafilter "^${target}" --filter \"$checkzfs_filter\" >> /tmp/${checkzfs_spool_maxage}_${checkzfs_prefix}"; fi
    $checkzfs $c_key $c_value --output checkmk --threshold $checkzfs_max_age --maxsnapshots $checkzfs_max_snapshot_count --prefix $checkzfs_prefix --replicafilter "^${target}" --filter "$checkzfs_filter" >> /tmp/${checkzfs_spool_maxage}_${checkzfs_prefix}
    if [ $checkzfs_spool -eq 0 ] || [ $checkzfs_local -gt 0 ]; then
        mkdir -p /var/lib/check_mk_agent/spool/
        $mv /tmp/${checkzfs_spool_maxage}_${checkzfs_prefix} /var/lib/check_mk_agent/spool/
    else
        $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "mkdir -p /var/lib/check_mk_agent/spool/"
        $scp -P ${sshport} /tmp/${checkzfs_spool_maxage}_${checkzfs_prefix} ${source}:/var/lib/check_mk_agent/spool/
    fi
fi

exit $rc