#!/usr/bin/env bash if [[ -n $ZIO_HELPERS_DIR ]] && [[ -d $ZIO_HELPERS_DIR ]]; then . "$ZIO_HELPERS_DIR/bash.sh" else if [[ "$(realpath "$(dirname "$(realpath -s "$0")")/../../../")" == "/" ]]; then . /usr/local/libexec/zio/helpers/bash.sh else . "$(dirname "$(realpath -s "$0")")/../libexec/zio/helpers/bash.sh" fi fi me_filename="$(basename "$(realpath -s "$0")")" backup_scripts_dir="$(get_config_dir "sh.zio.backup")/scripts" cache_dir="$(get_config_dir "sh.zio.backup" "/var/cache")/restic" secrets_dir="$(get_config_dir "sh.zio.backup")/secrets" host="$(hostname -s)" now="$(date +"%Y-%m-%d %H:%M:%S")" restic_mount_path="/mnt/restic" restic_path="" restic_repo_file="$secrets_dir/restic-repo" restic_repo_passwd_file="$secrets_dir/restic-repo-passwd" restic_version="0.16.0" function download_restic() { restic_version="$1" restic_download_url="https://github.com/restic/restic/releases/download/v${restic_version}/restic_${restic_version}_linux_amd64.bz2" restic_path="/tmp/restic-v$restic_version" restic_archive_path="${restic_path}_$(date +%s).${restic_download_url##*.}" if [[ ! -f "$restic_path" ]]; then say info "Downloading Restic ($restic_version)..." curl -L -s -o "$restic_archive_path" "$restic_download_url" bzip2 -dc "$restic_archive_path" > "$restic_path" rm -f "$restic_archive_path" chmod +x "$restic_path" if [[ ! "$(echo "$("$restic_path" version)")" == "restic $restic_version"* ]]; then die "Unexpected output from '$restic_path version'" rm -f "$restic_path" fi fi } function invoke_restic() { command="$1" args="${@:2}" case "$(cat "$restic_repo_file")" in "/"*) if [[ ! -d "$(cat "$restic_repo_file")" ]]; then say warning "'$(cat "$restic_repo_file")' does not exist. Creating directory..." mkdir -p "$(cat "$restic_repo_file")" fi ;; "b2"*) b2_account_id_file="$secrets_dir/b2-account-id" b2_account_key_file="$secrets_dir/b2-account-key" test_file "$b2_account_id_file" test_file "$b2_account_key_file" export B2_ACCOUNT_ID="$(cat "$b2_account_id_file")" export B2_ACCOUNT_KEY="$(cat "$b2_account_key_file")" ;; *) die "Repository unsupported ("$(cat "$restic_repo_file")")" ;; esac if [[ -z $command ]]; then say warning "No command specified. Not running" break elif [[ $command == "generate" || $command == "self-update" ]]; then say warning "Unsupported command: $command" break fi "$restic_path" \ --cache-dir "$cache_dir" \ --password-file "$restic_repo_passwd_file" \ --repo "$(cat $restic_repo_file)" \ $command $args } function invoke_script() { backup_script="$1" backup_script_filename="$(basename "$backup_script")" backup_script_name="${backup_script_filename%.*}" backup_script_name_length="${#backup_script_name}" say primary "-[$backup_script_name]$(repeat "-" $((80-3-$backup_script_name_length)))" chmod +x "$backup_script" export -f backup_dir export -f backup_dumps export -f backup_files export -f create_tmp_file export -f die export -f forget_backup export -f get_config_dir export -f get_dump_dir export -f get_real_path export -f get_secret export -f invoke_restic export -f podman_exec export -f prune_backup export -f prune_dumps export -f say export -f start_service export -f stop_service export -f test_file export backup_scripts_dir export cache_dir export host export me_filename export now export restic_repo_file export restic_repo_passwd_file export restic_path export secrets_dir set -o pipefail exec 3>&1 log_header="$me_filename $backup_script" log_header_length="${#log_header}" script_log_path="$(create_log "$log_header\n$(repeat "-" $log_header_length)")" script_error_log_path="$(create_tmp_file)" script_output=$("$backup_scripts_dir/$backup_script_filename" 2>$script_error_log_path | tee /dev/fd/3) script_result="$?" echo -e "$script_output" >> "$script_log_path" if [[ -f "$script_error_log_path" ]]; then cat "$script_error_log_path" cat "$script_error_log_path" >> "$script_log_path" fi if [[ "$script_result" == 0 ]]; then trigger_notify "success" "Backup succeeded: $backup_script_name" "$script_log_path" else trigger_notify "error" "Backup failed: $backup_script_name" "$(cat "$script_error_log_path" | sed -e "s/\`//g")" "$script_log_path" fi rm -f "$script_error_log_path" set +o pipefail exec 3>&- } function trigger_notify() { level="$1" title="$2" message="$3" log_path="$4" notify_prog="/usr/local/bin/sh.zio.notify" if [[ $log_path == "" ]]; then if [[ -f "$message" ]]; then log_path="$message" message="" fi fi if [[ ! -f "$notify_prog" ]]; then say warning "'$notify_prog' does not exist. Not sending notification" else if [[ "$message" == "" ]]; then "$notify_prog" \ --file "$log_path" \ --invoked-by "$(basename $me_filename)" \ --level "$level" \ --message "See attached log for details" \ --title "$title" elif [[ "$log_path" == "" ]]; then "$notify_prog" \ --invoked-by "$(basename $me_filename)" \ --level "$level" \ --message "$message" \ --title "$title" else message="$(echo "$3" | sed -r "s/\x1B\[[0-9;]*[JKmsu]//g")" "$notify_prog" \ --file "$log_path" \ --invoked-by "$(basename $me_filename)" \ --level "$level" \ --message '\\`\\`\\`\\n'"${message//$'\n'/'\\n'}"'\\n\\`\\`\\`' \ --title "$title" fi fi } function backup_files() { patterns="$1" args="${@:2}" files_from_path="$(create_tmp_file "files-from")" echo "$patterns" > "$files_from_path" sed -i -e 's/:/\n/g' "$files_from_path" say info "Backing up: $patterns ➔ $(cat $restic_repo_file)" invoke_restic \ backup \ --files-from "$files_from_path" \ --iexclude "__MACOSX" \ --iexclude ".cache" \ --iexclude ".DS_Store" \ --iexclude "cache" \ --iexclude "CachedData" \ --iexclude "CachedExtensionVSIXs" \ --iexclude "Code Cache" \ --iexclude "GPUCache" \ --iexclude "GrSharedCache" \ --iexclude "ShaderCache" \ --iexclude "system-cache" \ --iexclude "thumbs.db" \ --iexclude "tmp" \ --exclude "containers/storage/overlay" \ --exclude "containers/storage/overlay-containers" \ --exclude "containers/storage/overlay-images" \ --exclude "containers/storage/overlay-layers" \ --exclude-if-present ".nobackup" \ --host "$host" \ --tag "$me_filename" \ --tag "$(basename "$0")" \ $args rm -f "$files_from_path" } function backup_dir() { path="$1" args="${@:2}" if [[ ! -d "$path" ]]; then say warning "'$path' does not exist. Not backing up" else backup_files "$path" $args fi } function backup_dumps() { service="$1" service_dumps_dir="/srv/dumps/$host/$service" backup_dir "$service_dumps_dir" if [[ $? == 0 ]]; then rm -rf "$service_dumps_dir" prune_dumps fi } function forget_backup() { timeframe="$1" if [[ -z $timeframe ]]; then timeframe="0y0m7d0h" fi say info "Forgetting: $timeframe ($host)" invoke_restic \ forget \ --compact \ --keep-within "$timeframe" \ --host "$host" } function prune_backup() { say info "Pruning" invoke_restic \ prune \ --dry-run } function prune_dumps() { [[ -z "$(ls -A "/srv/dumps/$host")" ]] && rm -rf "/srv/dumps/$host" [[ -z "$(ls -A "/srv/dumps")" ]] && rm -rf "/srv/dumps" } function get_dump_dir() { service="$1" specific_dumps_dir="/srv/dumps/$host/$service/$(date +"%Y%m%d%H%M%S")" mkdir -p "$specific_dumps_dir" [[ $? == "0" ]] && echo "$specific_dumps_dir" } function get_secret() { secret_path="$secrets_dir/$1" if [[ -f "$secret_path" ]]; then cat "$secret_path" fi } function print_time() { ((h=${1}/3600)) ((m=(${1}%3600)/60)) ((s=${1}%60)) h_string="hours" m_string="minutes" s_string="seconds" [[ $h == 1 ]] && h_string="hour" [[ $m == 1 ]] && m_string="minute" [[ $s == 1 ]] && s_string="second" output="" [[ $h != "0" ]] && output+="$h $h_string" [[ $m != "0" ]] && output+=" $m $m_string" [[ $s != "0" ]] && output+=" $s $s_string" echo $output } function start_service() { service="$1" say info "Starting service: $service" systemctl start $service } function stop_service() { service="$1" say info "Stopping service: $service" systemctl stop $service } if [[ "$@" == "help" ]]; then echo "$me_filename Usage: $me_filename Run all backup scripts available in $backup_scripts_dir $me_filename