Scripts For backup

srv/exo/bin/_cleanBackups.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+
+echo "[INFO] ======================================="
+echo "[INFO] = Remove old backups and keep only the ${BACKUP_RETENTION} newest ones ..."
+echo "[INFO] ======================================="
+
+pushd ${BACKUP_DIR} &>/dev/null
+
+NB_FILES_PER_BACKUP=4
+
+LINES=$((BACKUP_RETENTION * NB_FILES_PER_BACKUP + 1 ))
+rm -rvf $(find . -maxdepth 1 -type f | xargs ls -t | tail -n +${LINES})
+
+echo "[INFO] Done"
+popd &>/dev/null

srv/exo/bin/_dumpData.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+# Initialize working directory
+mkdir -p ${BACKUP_WORKING_DIR}/tmp_data
+rm -rf ${BACKUP_WORKING_DIR}/tmp_data/*
+pushd ${BACKUP_WORKING_DIR}/tmp_data >/dev/null 2>&1
+
+echo "[INFO] ======================================="
+echo "[INFO] = Compressing ${PLF_NAME} data into ${BACKUP_WORKING_DIR}/tmp_data/${PLF_NAME}-data-${BACKUP_DATE}.tar.bz2 ..."
+echo "[INFO] ======================================="
+echo "[INFO] $(display_date)"
+
+pushd ${BACKUP_WORKING_DIR}/tmp_data >/dev/null 2>&1
+
+display_time tar --directory ${EXO_DATA_DIR} --use-compress-prog=pbzip2 -cpf ${BACKUP_WORKING_DIR}/tmp_data/${PLF_NAME}-data-${BACKUP_DATE}.tar.bz2 .
+
+popd >/dev/null 2>&1
+echo "[INFO] Done"

srv/exo/bin/_dumpElasticsearch.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+# Initialize working directory
+mkdir -p ${BACKUP_WORKING_DIR}/tmp_elasticsearch
+rm -rf ${BACKUP_WORKING_DIR}/tmp_elasticsearch/*
+pushd ${BACKUP_WORKING_DIR}/tmp_elasticsearch >/dev/null 2>&1
+
+echo "[INFO] ======================================="
+echo "[INFO] = Compressing ${PLF_NAME} data into ${BACKUP_WORKING_DIR}/tmp_elasticsearch/${PLF_NAME}-es-${BACKUP_DATE}.tar.bz2 ..."
+echo "[INFO] ======================================="
+echo "[INFO] $(display_date)"
+
+pushd ${BACKUP_WORKING_DIR}/tmp_elasticsearch >/dev/null 2>&1
+
+display_time tar --directory $(dirname ${ELASTICSEARCH_DATA_DIR}) --use-compress-prog=pbzip2 -cpf ${BACKUP_WORKING_DIR}/tmp_elasticsearch/${PLF_NAME}-es-${BACKUP_DATE}.tar.bz2 $(basename ${ELASTICSEARCH_DATA_DIR})
+
+popd >/dev/null 2>&1
+echo "[INFO] Done"

srv/exo/bin/_dumpMongoDb.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+# Initialize working directory
+mkdir -p ${BACKUP_WORKING_DIR}/tmp_mongo
+rm -rf ${BACKUP_WORKING_DIR}/tmp_mongo/*
+pushd ${BACKUP_WORKING_DIR}/tmp_mongo/ >/dev/null 2>&1
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+# settings : http://docs.mongodb.org/manual/single/index.html#document-tutorial/backup-databases-with-binary-database-dumps
+
+echo "[INFO] $(display_date)"
+echo "[INFO] ======================================="
+echo "[INFO] = Dumping MongoDB ${CHAT_DATABASE} into ${PLF_NAME}-${CHAT_DATABASE}-${BACKUP_DATE}.tar.bz2 ..."
+echo "[INFO] ======================================="
+echo "[INFO] $(display_date)"
+
+TARGET="${PLF_NAME}-${CHAT_DATABASE}-${BACKUP_DATE}.tar.bz2"
+
+display_time mongodump -o ${BACKUP_WORKING_DIR}/tmp_mongo --db=${CHAT_DATABASE}
+
+echo "[INFO] Compress mongo dump into ${BACKUP_WORKING_DIR}/tmp_mongo/${TARGET}"
+display_time tar --directory ${BACKUP_WORKING_DIR}/tmp_mongo --use-compress-prog=pbzip2 -cpf ${BACKUP_WORKING_DIR}/tmp_mongo/${TARGET} ${CHAT_DATABASE}
+
+echo "[INFO] Remove ${BACKUP_WORKING_DIR}/tmp_mongo/${CHAT_DATABASE}"
+display_time rm -rf ${BACKUP_WORKING_DIR}/tmp_mongo/${CHAT_DATABASE}
+
+echo "[INFO] Done"
+echo "[INFO] $(display_date)"
+
+popd >/dev/null 2>&1

srv/exo/bin/_dumpMysqlDatabase.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+DUMP_OPTIONS="--single-transaction --add-drop-table"
+
+# Initialize working directory
+mkdir -p ${BACKUP_WORKING_DIR}/tmp_db
+rm -rf ${BACKUP_WORKING_DIR}/tmp_db/*
+pushd ${BACKUP_WORKING_DIR}/tmp_db >/dev/null 2>&1
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+TARGET_NAME="${PLF_NAME}-db-${BACKUP_DATE}.sql.bz2"
+
+echo "[INFO] ======================================="
+echo "[INFO] = Dumping database ${EXO_DATABASE} into ${BACKUP_WORKING_DIR}/tmp_db/${TARGET_NAME} ..."
+echo "[INFO] ======================================="
+echo "[INFO] $(display_date)"
+
+display_time mysqldump ${DUMP_OPTIONS} ${EXO_DATABASE} | pbzip2 >${BACKUP_WORKING_DIR}/tmp_db/${TARGET_NAME}
+
+echo "[INFO] Done"
+
+popd >/dev/null 2>&1

srv/exo/bin/_functions.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -eu
+
+#Activate aliases usage in scripts
+shopt -s expand_aliases
+
+# Various command aliases
+alias display_time='/usr/bin/time -f "[INFO] Return code : %x\n[INFO] Time report (sec) : \t%e real,\t%U user,\t%S system"'
+alias display_date='/bin/date +"%Y-%m-%d %H:%M:%S"'
+
+getSSHCommand() {
+  if [ $# -ne 2 ]; then
+    echo ""
+    echo "[ERROR] No enough parameters for function getSSHCommand"
+    echo "[ERROR] The target host and the user must be specified"
+    exit 1
+  fi
+  TARGET=$1
+  USER=$2
+
+  if [ "${TARGET}" == "localhost" ] || [ "${TARGET}" == "127.0.0.1" ]; then
+    echo ""
+  else
+   echo "ssh ${USER}@${TARGET}" # space is important here
+  fi
+}
+
+# $1 : Startup time
+# $2 : End time
+delay() {
+  if [ $# -lt 2 ]; then
+    echo ""
+    echo "[ERROR] No enough parameters for function delay !"
+    exit 1
+  fi
+  local _start=$1
+  local _end=$2
+  echo "$(((_end - _start) / 3600)) hour(s) $((((_end - _start) % 3600) / 60)) minute(s) $(((_end - _start) % 60)) second(s)"
+}
+
+# $1 : Message
+# $2 : Startup time
+# $3 : End time
+display_delay() {
+  if [ $# -lt 3 ]; then
+    echo ""
+    echo "[ERROR] No enough parameters for function display_delay !"
+    exit 1
+  fi
+  echo "[INFO] $1: $(delay $2 $3) ."
+}
+
+# $1 : Action
+# $2 : Service
+systemd_action() {
+  if [ $# -lt 2 ]; then
+    echo ""
+    echo "[ERROR] No enough parameters for function systemd_action !"
+    exit 1
+  fi
+  local ACTION=$1
+  local SERVICE=$2
+
+  # check is the service is installed
+  if [ "$(systemctl list-unit-files ${SERVICE}.service --quiet --no-legend | wc -l)" == 0 ]; then
+    echo "[ERROR] Service ${SERVICE} not found"
+    exit 1
+  fi
+
+  case ${ACTION} in
+  start)
+    sudo systemctl start ${SERVICE}
+    if $(sudo systemctl -q is-active ${SERVICE}); then
+      echo "[INFO] Service ${SERVICE} started successfuly"
+    else
+      echo "[ERROR] Service ${SERVICE} failed to start"
+      exit 1
+    fi
+    ;;
+  stop)
+    echo "[INFO] Stop ${SERVICE} service"
+    sudo systemctl stop ${SERVICE}
+    ;;
+  status)
+    sudo systemctl status ${SERVICE}
+    break
+    ;;
+  *)
+    echo "[ERROR] No systemd action defined !"
+    exit 1
+    ;;
+  esac
+
+}

srv/exo/bin/_restoreData.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+# Initialize working directory
+pushd ${EXO_DATA_DIR} >/dev/null 2>&1
+
+ARCHIVE="${BACKUP_WORKING_DIR}/tmp_data/${PLF_NAME}-data-${BACKUP_DATE}.tar.bz2"
+
+echo "[INFO] ======================================="
+echo "[INFO] Restoring data from ${ARCHIVE}"
+echo "[INFO] ======================================="
+echo "[INFO] $(display_date)"
+
+if ${BACKUP_ON_RESTORE}; then
+  echo "[INFO] Saving current data ..."
+
+  if [ -e ${EXO_DATA_DIR}/data.beforerestore ]; then
+    mv -v ${EXO_DATA_DIR}/data.beforerestore ${EXO_DATA_DIR}/data.beforerestore-old
+    # Can be long process in background
+    rm -rf ${EXO_DATA_DIR}/data.beforerestore-old &
+  fi
+
+  if [ -e ${EXO_DATA_DIR}/data ]; then
+    mv -v ${EXO_DATA_DIR}/data ${EXO_DATA_DIR}/data.beforerestore
+  fi
+else
+  echo "[INFO] Removing current data ..."
+  if [ -e ${EXO_DATA_DIR}/data ]; then
+    mv -v ${EXO_DATA_DIR}/data ${EXO_DATA_DIR}/data.beforerestore
+    # Can be long process in background
+    rm -rf ${EXO_DATA_DIR}/data.beforerestore &
+  fi
+fi
+
+pushd ${BACKUP_WORKING_DIR}/tmp_data >/dev/null 2>&1
+
+echo "[INFO] = Uncompressing ${ARCHIVE} into ${EXO_DATA_DIR} ..."
+display_time tar xf ${ARCHIVE} -C ${EXO_DATA_DIR}
+echo "[INFO] $(display_date)"
+echo "[INFO] Done"
+
+rm -v ${ARCHIVE}
+
+echo "[INFO] = waiting for cleanup to end ..."
+# be sure all the cleanup processes are finished
+wait 
+
+popd >/dev/null 2>&1
+echo "[INFO] Done"

srv/exo/bin/_restoreElasticsearch.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -eu
+
+# #############################################################################
+# Initialize
+# #############################################################################
+SCRIPT_NAME="${0##*/}"
+SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Load env settings
+source ${SCRIPT_DIR}/setenv.sh
+# Load common functions
+source ${SCRIPT_DIR}/_functions.sh
+
+BACKUP_DATE=${1:-$(date "+%Y-%m-%d-%H%M%S")}
+
+echo "[INFO] ======================================="
+echo "[INFO] Restoring Elasticsearch data"
+echo "[INFO] ======================================="
+
+pushd ${BACKUP_WORKING_DIR}/tmp_elasticsearch >/dev/null 2>&1
+
+if ${BACKUP_ON_RESTORE} && [ -e "${ELASTICSEARCH_DATA_DIR}/nodes.old" ]; then
+  echo "[INFO] Removing previous backup save directory (${ELASTICSEARCH_DATA_DIR}/nodes.old)..."
+  sudo -u elasticsearch rm -rf ${ELASTICSEARCH_DATA_DIR}/nodes.old
+fi
+
+if ${BACKUP_ON_RESTORE} && [ -e "${ELASTICSEARCH_DATA_DIR}/nodes" ]; then
+  echo "[INFO] Keeping current data in (${ELASTICSEARCH_DATA_DIR}/nodes.old) directory ..."
+  sudo -u elasticsearch mv -f ${ELASTICSEARCH_DATA_DIR}/nodes ${ELASTICSEARCH_DATA_DIR}/nodes.old
+else 
+  echo "[INFO] Removing current data on (${ELASTICSEARCH_DATA_DIR}/nodes) directory ..."
+  sudo -u elasticsearch rm -rf ${ELASTICSEARCH_DATA_DIR}/nodes ${ELASTICSEARCH_DATA_DIR}/nodes
+fi
+
+echo "[INFO] = Uncompressing ${BACKUP_WORKING_DIR}/tmp_elasticsearch/${PLF_NAME}-es-${BACKUP_DATE}.tar.bz2 into ${ELASTICSEARCH_DATA_DIR}  ..."
+echo "[INFO] $(display_date)"
+display_time sudo -u elasticsearch tar xf ${BACKUP_WORKING_DIR}/tmp_elasticsearch/${PLF_NAME}-es-${BACKUP_DATE}.tar.bz2 -C ${ELASTICSEARCH_DATA_DIR}/..
+
+popd >/dev/null 2>&1
+echo "[INFO] Done"