#1 consolidated prom push

restic/README.md

@@ -18,6 +18,10 @@ docker logs restic -f
 
 ### Operation
 
+#### Monitoring
+For monitoring purposes the script `data/restic/push-metric.sh` is executed prior to and after any backup.
+If the environment variable `$PROM_GW_URL` is set the respective Prometheus Push Gateway will get informed about the backup start and result (stop, incomplete, fail). In case of a proper stop, further statistics will be send to the Push Gateway.
+
 #### List snapshots with date/time
 ```
 / # restic ls

restic/data/restic/notify-prom-stats.sh

@@ -1,15 +0,0 @@
-#!/bin/sh -xe
-
-json=`restic stats --json`
-
-[ ! -z $PROM_GW_URL ] && cat <<EOF | curl --data-binary @- $PROM_GW_URL 
-# TYPE restic_repository_total_size_bytes gauge
-# HELP restic_repository_total_size_bytes total size of the repository
-`echo -n 'restic_repository_total_size_bytes ' && \
-echo $json | grep -o '"total_size":[0-9]*' | grep -o '[0-9]*'`
-
-# TYPE restic_repository_total_files_count gauge
-# HELP restic_repository_total_files_count total count of files in the repository
-`echo -n 'restic_repository_total_files_count ' && \
-echo $json | grep -o '"total_file_count":[0-9]*' | grep -o '[0-9]*'`
-EOF

restic/data/restic/notify-prom-status.sh

@@ -1,16 +0,0 @@
-#!/bin/sh -xe
-
-case $1 in
-  start)      state='1';  status='started'    ;;
-  stop)       state='0';  status='stopped'    ;;
-  incomplete) state='-3'; status='incomplete' ;;
-  fail|*)     state='-1'; status='failed'     ;;
-esac
-
-[ ! -z $PROM_GW_URL ] && cat <<EOF | curl --data-binary @- $PROM_GW_URL 
-# TYPE restic_hook_state gauge
-# HELP restic_hook_state 1=started 0=stopped -1=failed -3=incomplete
-restic_hook_state $state
-# TYPE restic_hook_state_timestamp gauge
-restic_hook_state_timestamp{label="$status"} `date +%s`
-EOF

restic/data/restic/push-metric.sh

@@ -0,0 +1,47 @@
+#!/bin/sh -xe
+
+if [ $# -eq 0 ]; then
+  echo 'Script to update a Prometheus Pushgateway with restic metrics'
+  echo 'Syntax: push-metric.sh [start|stop|incomplete|fail]'
+  echo 'Gateway-URL must be set as environment variable in $PROM_GW_URL'
+  exit 0
+fi
+
+if [ -z $PROM_GW_URL ]; then
+  echo 'No Prometheus Push Gateway set in $PROM_GW_URL'
+  exit 0 
+fi
+
+## Set numeric state value and verbal status based on the received event
+case $1 in
+  start)      state='1';  status='started'    ;;
+  stop)       state='0';  status='stopped'    ;;
+  incomplete) state='-3'; status='incomplete' ;;
+  fail|*)     state='-1'; status='failed'     ;;
+esac
+
+## Update the Prometheus Pushgateway about the state/status
+cat <<EOF | curl --data-binary @- $PROM_GW_URL 
+# TYPE restic_hook_state gauge
+# HELP restic_hook_state 1=started 0=stopped -1=failed -3=incomplete
+restic_hook_state $state
+# TYPE restic_hook_state_timestamp gauge
+restic_hook_state_timestamp{label="$status"} `date +%s`
+EOF
+
+## Update the Prometheus Pushgateway about statistics if status is stopped
+if [ $status = 'stopped' ]; then
+  json=`restic stats --json`
+
+  cat <<EOF | curl --data-binary @- $PROM_GW_URL 
+# TYPE restic_repository_total_size_bytes gauge
+# HELP restic_repository_total_size_bytes total size of the repository
+`echo -n 'restic_repository_total_size_bytes ' && \
+echo $json | grep -o '"total_size":[0-9]*' | grep -o '[0-9]*'`
+
+# TYPE restic_repository_total_files_count gauge
+# HELP restic_repository_total_files_count total count of files in the repository
+`echo -n 'restic_repository_total_files_count ' && \
+echo $json | grep -o '"total_file_count":[0-9]*' | grep -o '[0-9]*'`
+EOF
+fi

restic/data/restic/run.sh

@@ -3,18 +3,18 @@
 # Init restic repository if neccessary
 restic snapshots &>/dev/null || restic init
 
-# Notify prometheus on backup start if $PROM_GW_URL is set
-/root/restic/notify-prom-status.sh start
+# Monitoring: Set metric to started
+/root/restic/push-metric.sh start
 
 # Run backup job, using parameters from AWS_ACCESS_KEY_ID,
 # AWS_SECRET_ACCESS_KEY, RESTIC_PASSWORD, RESTIC_REPOSITORY
 restic backup $RESTIC_BACKUP_ARGS $RESTIC_BACKUP_FOLDERS
 
+# Monitoring: Set metric based on the result code of the backup job
 case $? in
-  0)   /root/restic/notify-prom-status.sh stop
-       /root/restic/notify-prom-stats.sh             ;;
-  3)   /root/restic/notify-prom-status.sh incomplete ;;
-  1|*) /root/restic/notify-prom-status.sh fail       ;;
+  0)   /root/restic/push-metric.sh stop ;;
+  3)   /root/restic/push-metric.sh incomplete ;;
+  1|*) /root/restic/push-metric.sh fail       ;;
 esac
   
 # Cleanup old backups