#!/usr/bin/env bash if [[ -n $ZIO_HELPERS_DIR ]] && [[ -d $ZIO_HELPERS_DIR ]]; then . "$ZIO_HELPERS_DIR/bash.sh" else if [[ "$(realpath "$(dirname "$(realpath -s "$0")")/../../../")" == "/" ]]; then . /usr/local/libexec/zio/helpers/bash.sh else . "$(dirname "$(realpath -s "$0")")/../libexec/zio/helpers/bash.sh" fi fi me_filename="$(basename "$(realpath -s "$0")")" backup_scripts_dir="$(get_config_dir "sh.zio.backup")/scripts" cache_dir="$(get_config_dir "sh.zio.backup" "/var/cache")/restic" secrets_dir="$(get_config_dir "sh.zio.backup")/secrets" host="$(hostname -s)" now="$(date +"%Y-%m-%d %H:%M:%S")" restic_mount_path="/mnt/restic" restic_path="" restic_repo_file="$secrets_dir/restic-repo" restic_repo_passwd_file="$secrets_dir/restic-repo-passwd" restic_version="0.16.0" function download_restic() { restic_version="$1" restic_download_url="https://github.com/restic/restic/releases/download/v${restic_version}/restic_${restic_version}_linux_amd64.bz2" restic_path="/tmp/restic-v$restic_version" restic_archive_path="${restic_path}_$(date +%s).${restic_download_url##*.}" if [[ ! -f "$restic_path" ]]; then say info "Downloading Restic ($restic_version)..." curl -L -s -o "$restic_archive_path" "$restic_download_url" bzip2 -dc "$restic_archive_path" > "$restic_path" rm -f "$restic_archive_path" chmod +x "$restic_path" if [[ ! "$(echo "$("$restic_path" version)")" == "restic $restic_version"* ]]; then die "Unexpected output from '$restic_path version'" rm -f "$restic_path" fi fi } function invoke_restic() { command="$1" args="${@:2}" case "$(cat "$restic_repo_file")" in "/"*) if [[ ! -d "$(cat "$restic_repo_file")" ]]; then say warning "'$(cat "$restic_repo_file")' does not exist. Creating directory..." mkdir -p "$(cat "$restic_repo_file")" fi ;; "b2"*) b2_account_id_file="$secrets_dir/b2-account-id" b2_account_key_file="$secrets_dir/b2-account-key" test_file "$b2_account_id_file" test_file "$b2_account_key_file" export B2_ACCOUNT_ID="$(cat "$b2_account_id_file")" export B2_ACCOUNT_KEY="$(cat "$b2_account_key_file")" ;; *) die "Repository unsupported ("$(cat "$restic_repo_file")")" ;; esac if [[ -z $command ]]; then say warning "No command specified. Not running" break elif [[ $command == "generate" || $command == "self-update" ]]; then say warning "Unsupported command: $command" break fi "$restic_path" \ --cache-dir "$cache_dir" \ --password-file "$restic_repo_passwd_file" \ --repo "$(cat $restic_repo_file)" \ $command $args #attempts=10 #attempts_delay=600 #while true; do #set -o pipefail #exec 3>&1 #function trigger_ntfy() { # level="$1" # title="$2" # output="$3" # /usr/local/bin/sh.zio.notify # --invoked-by "sh.zio.backup" \ # --level "$level" \ # --message '```\\n'"${output//$'\n'/'\\n'}"'\\n```' \ # --title "Backup failed" #} #restic_out="$("$restic_path" \ # --cache-dir "$cache_dir" \ # --password-file "$restic_repo_passwd_file" \ # --repo "$(cat $restic_repo_file)" \ # $command $args 2>&1 | tee /dev/fd/3)" #else #say warning "Command failed." #echo "$restic_out" #/usr/local/bin/sh.zio.notify \ # --title "Backup failed" \ # --message '```\\n'"${restic_out//$'\n'/'\\n'}"'\\n```' \ # --level "warning" \ # --invoked-by "sh.zio.backup" #trigger_ntfy "warning" "Backup failed: " #break #echo "---- THE OUTPUT: $restic_out" #if [[ $attempts == 0 ]]; then # say warning "Command failed. No attempts left" # break #else # attempts=$((attempts - 1)) # say warning "Command failed. Trying again in $attempts_delay seconds ($attempts attempts remaining)..." # sleep $attempts_delay #fi #fi #set +o pipefail #exec 3>&- #done #fi } function invoke_script() { backup_script="$1" backup_script_filename="$(basename "$backup_script")" backup_script_name="${backup_script_filename%.*}" backup_script_name_length="${#backup_script_name}" say primary "-[$backup_script_name]$(repeat "-" $((80-3-$backup_script_name_length)))" chmod +x "$backup_script" export -f backup_dir export -f backup_dumps export -f backup_files export -f create_tmp_file export -f die export -f forget_backup export -f get_config_dir export -f get_dump_dir export -f get_real_path export -f get_secret export -f invoke_restic export -f podman_exec export -f prune_backup export -f prune_dumps export -f say export -f start_service export -f stop_service export -f test_file export backup_scripts_dir export cache_dir export host export me_filename export now export restic_repo_file export restic_repo_passwd_file export restic_path export secrets_dir set -o pipefail exec 3>&1 script_output=$("$backup_scripts_dir/$backup_script_filename" 2>&1 | tee /dev/fd/3) if [[ "$?" == 0 ]]; then trigger_notify "success" "Backup Successful: $backup_script_name" "$script_output" else trigger_notify "error" "Backup Failed: $backup_script_name" "$script_output" fi set +o pipefail exec 3>&- } function trigger_notify() { level="$1" title="$2" output="$3" notify_prog="/usr/local/bin/sh.zio.notify" if [[ ! -f "$notify_prog" ]]; then say warning "'$notify_prog' not found. Not sending notification" else /usr/local/bin/sh.zio.notify \ --invoked-by "sh.zio.backup" \ --level "$level" \ --message '```\\n'"${output//$'\n'/'\\n'}"'\\n```' \ --title "$title" fi } function backup_files() { patterns="$1" args="${@:2}" files_from_path="$(create_tmp_file "files-from")" echo "$patterns" > "$files_from_path" sed -i -e 's/:/\n/g' "$files_from_path" say info "Backing up: $patterns ➔ $(cat $restic_repo_file)" invoke_restic \ backup \ --files-from "$files_from_path" \ --iexclude "__MACOSX" \ --iexclude ".cache" \ --iexclude ".DS_Store" \ --iexclude "cache" \ --iexclude "CachedData" \ --iexclude "CachedExtensionVSIXs" \ --iexclude "Code Cache" \ --iexclude "GPUCache" \ --iexclude "GrSharedCache" \ --iexclude "ShaderCache" \ --iexclude "system-cache" \ --iexclude "thumbs.db" \ --iexclude "tmp" \ --exclude "containers/storage/overlay" \ --exclude "containers/storage/overlay-containers" \ --exclude "containers/storage/overlay-images" \ --exclude "containers/storage/overlay-layers" \ --exclude-if-present ".nobackup" \ --host "$host" \ --tag "$me_filename" \ --tag "$(basename "$0")" \ $args rm -f "$files_from_path" } function backup_dir() { path="$1" args="${@:2}" if [[ ! -d "$path" ]]; then say warning "'$path' does not exist. Not backing up" else backup_files "$path" $args fi } function backup_dumps() { service="$1" service_dumps_dir="/srv/dumps/$host/$service" backup_dir "$service_dumps_dir" if [[ $? == 0 ]]; then rm -rf "$service_dumps_dir" prune_dumps fi } function forget_backup() { timeframe="$1" if [[ -z $timeframe ]]; then timeframe="0y0m7d0h" fi say info "Forgetting: $timeframe ($host)" invoke_restic \ forget \ --compact \ --keep-within "$timeframe" \ --host "$host" } function prune_backup() { say info "Pruning" invoke_restic \ prune \ --dry-run } function prune_dumps() { [[ -z "$(ls -A "/srv/dumps/$host")" ]] && rm -rf "/srv/dumps/$host" [[ -z "$(ls -A "/srv/dumps")" ]] && rm -rf "/srv/dumps" } function get_dump_dir() { service="$1" specific_dumps_dir="/srv/dumps/$host/$service/$(date +"%Y%m%d%H%M%S")" mkdir -p "$specific_dumps_dir" [[ $? == "0" ]] && echo "$specific_dumps_dir" } function get_secret() { secret_path="$secrets_dir/$1" if [[ -f "$secret_path" ]]; then cat "$secret_path" fi } function start_service() { service="$1" say info "Starting service: $service" systemctl start $service } function stop_service() { service="$1" say info "Stopping service: $service" systemctl stop $service } if [[ "$@" == "help" ]]; then echo "$me_filename Usage: $me_filename Run all backup scripts available in $backup_scripts_dir $me_filename