Feature restructure backup

packages/helpermodules/subdata.py

@@ -898,9 +898,11 @@ def process_system_topic(self, client: mqtt.Client, var: dict, msg: mqtt.MQTTMes
                 self.set_json_payload(var["system"].data["backup_cloud"], msg)
             elif ("openWB/system/dataprotection_acknowledged" == msg.topic and
                     decode_payload(msg.payload) is False):
-                Pub().pub("openWB/set/command/removeCloudBridge/todo", {
-                    "command": "removeCloudBridge"
-                })
+                if self.event_subdata_initialized.is_set():
+                    Pub().pub("openWB/set/command/removeCloudBridge/todo",
+                              {"command": "removeCloudBridge"})
+                else:
+                    log.debug("skipping data protection message on startup")
             elif re.search("^.+/io/[0-9]+/config$", msg.topic) is not None:
                 index = get_index(msg.topic)
                 if decode_payload(msg.payload) == "":

runs/atreboot.sh

@@ -398,8 +398,8 @@ chmod 666 "$LOGFILE"
 	fi
 
 	# set restore dir permissions to allow file upload for apache
-	sudo chgrp www-data "${OPENWBBASEDIR}/data/restore" "${OPENWBBASEDIR}/data/restore/"* "${OPENWBBASEDIR}/data/data_migration" "${OPENWBBASEDIR}/data/data_migration/"*
-	sudo chmod g+w "${OPENWBBASEDIR}/data/restore" "${OPENWBBASEDIR}/data/restore/"* "${OPENWBBASEDIR}/data/data_migration" "${OPENWBBASEDIR}/data/data_migration/"*
+	sudo chgrp -R www-data "${OPENWBBASEDIR}/data/restore/." "${OPENWBBASEDIR}/data/data_migration/."
+	sudo chmod -R g+w "${OPENWBBASEDIR}/data/restore/." "${OPENWBBASEDIR}/data/data_migration/."
 
 	# cleanup some folders
 	folder="${OPENWBBASEDIR}/data/data_migration/var"

runs/backup.sh

@@ -4,7 +4,10 @@ OPENWBDIRNAME=${OPENWBBASEDIR##*/}
 OPENWBDIRNAME=${OPENWBDIRNAME:-/}
 TARBASEDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 BACKUPDIR="$OPENWBBASEDIR/data/backup"
-LOGFILE="$OPENWBBASEDIR/data/log/backup.log"
+RAMDISKDIR="$OPENWBBASEDIR/ramdisk"
+TEMPDIR="$RAMDISKDIR/temp"
+LOGDIR="$OPENWBBASEDIR/data/log"
+LOGFILE="$LOGDIR/backup.log"
 
 useExtendedFilename=$1
 if ((useExtendedFilename == 1)); then
@@ -16,6 +19,18 @@ else
 fi
 
 {
+	echo "starting backup script"
+	echo "environment:"
+	echo "  OPENWBBASEDIR: $OPENWBBASEDIR"
+	echo "  OPENWBDIRNAME: $OPENWBDIRNAME"
+	echo "  TARBASEDIR: $TARBASEDIR"
+	echo "  BACKUPDIR: $BACKUPDIR"
+	echo "  RAMDISKDIR: $RAMDISKDIR"
+	echo "  TEMPDIR: $TEMPDIR"
+	echo "  LOGDIR: $LOGDIR"
+	echo "  LOGFILE: $LOGFILE"
+	echo "  FILENAME: $FILENAME"
+
 	echo "deleting old backup files if present in '$BACKUPDIR'"
 	# remove old backup files
 	rm -v "$BACKUPDIR/"*
@@ -33,13 +48,10 @@ fi
 	tar --verbose --create \
 		--file="$BACKUPFILE" \
 		--directory="$TARBASEDIR/" \
-		--exclude="$OPENWBDIRNAME/data/backup/*.tar" \
-		--exclude="$OPENWBDIRNAME/data/log/backup.log" \
-		--exclude="$OPENWBDIRNAME/.git" \
-		--exclude "$OPENWBDIRNAME/ramdisk" \
-		--exclude "__pycache__" \
-		--exclude "$OPENWBDIRNAME/.pytest_cache" \
-		"$OPENWBDIRNAME"
+		"$OPENWBDIRNAME/data/charge_log" \
+		"$OPENWBDIRNAME/data/daily_log" \
+		"$OPENWBDIRNAME/data/monthly_log" \
+		"$OPENWBDIRNAME/data/log/uuid"
 	echo "adding configuration file"
 	sudo tar --verbose --append \
 		--file="$BACKUPFILE" \
@@ -50,46 +62,54 @@ fi
 		--file="$BACKUPFILE" \
 		--directory="/var/lib/" \
 		"mosquitto/" "mosquitto_local/"
+	echo "adding mosquitto configuration"
+	sudo tar --verbose --append \
+		--file="$BACKUPFILE" \
+		--directory="/etc/mosquitto/" \
+		"conf_local.d/"
+	echo "adding boot file"
+	sudo tar --verbose --append \
+		--file="$BACKUPFILE" \
+		"/boot/config.txt"
 	echo "adding git information"
-	git branch --no-color --show-current >"$OPENWBBASEDIR/ramdisk/GIT_BRANCH"
-	git log --pretty='format:%H' -n1 >"$OPENWBBASEDIR/ramdisk/GIT_HASH"
-	echo "branch: $(<"$OPENWBBASEDIR/ramdisk/GIT_BRANCH") commit-hash: $(<"$OPENWBBASEDIR/ramdisk/GIT_HASH")"
+	git branch --no-color --show-current >"$RAMDISKDIR/GIT_BRANCH"
+	git log --pretty='format:%H' -n1 >"$RAMDISKDIR/GIT_HASH"
+	echo "branch: $(<"$RAMDISKDIR/GIT_BRANCH") commit-hash: $(<"$RAMDISKDIR/GIT_HASH")"
 	tar --verbose --append \
 		--file="$BACKUPFILE" \
-		--directory="$OPENWBBASEDIR/ramdisk/" \
+		--directory="$RAMDISKDIR/" \
 		"GIT_BRANCH" "GIT_HASH"
 
 	echo "calculating checksums"
-	# openwb directory
-	find "$OPENWBBASEDIR" \( \
-		-path "$OPENWBBASEDIR/.git" -o \
-		-path "$OPENWBBASEDIR/data/backup" -o \
-		-path "$OPENWBBASEDIR/.pytest" -o \
-		-name "__pycache__" -o \
-		-path "$OPENWBBASEDIR/.pytest_cache" -o \
-		-path "$OPENWBBASEDIR/ramdisk" -o \
-		-name "backup.log" \
-		\) -prune -o \
-		-type f -print0 | xargs -0 sha256sum | sed -n "s|$TARBASEDIR/||p" >"$OPENWBBASEDIR/ramdisk/SHA256SUM"
-	# configuration file
-	echo -n "/home/openwb/configuration.json" | xargs -0 sha256sum | sed -n "s|/home/openwb/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM"
-	# git info files
-	find "$OPENWBBASEDIR/ramdisk/GIT_"* \
-		-type f -print0 | xargs -0 sha256sum | sed -n "s|$OPENWBBASEDIR/ramdisk/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM"
-	# mosquitto databases
-	find "/var/lib/mosquitto"* \
-		-type f -print0 | xargs -0 sudo sha256sum | sed -n "s|/var/lib/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM"
+	IFS=$'\n'
+	mapfile -t file_list < <(tar -tf "$BACKUPFILE")
+	mkdir -p "$TEMPDIR"
+	# process each file
+	for file in "${file_list[@]}"; do
+		# skip directories
+		if [[ $file =~ /$ ]]; then
+			echo "skipping directory $file"
+			continue
+		fi
+		# extract the file
+		tar -xf "$BACKUPFILE" -C "$TEMPDIR" "$file"
+		# calculate the checksum
+		sha256sum "$TEMPDIR/$file" | sed -n "s|$TEMPDIR/||p" >> "$RAMDISKDIR/SHA256SUM"
+		# remove the file
+		rm -f "$TEMPDIR/$file"
+	done
 	tar --verbose --append \
 		--file="$BACKUPFILE" \
-		--directory="$OPENWBBASEDIR/ramdisk/" \
+		--directory="$RAMDISKDIR/" \
 		"SHA256SUM"
 
 	# cleanup
 	echo "removing temporary files"
-	rm -v "$OPENWBBASEDIR/ramdisk/GIT_BRANCH" "$OPENWBBASEDIR/ramdisk/GIT_HASH" "$OPENWBBASEDIR/ramdisk/SHA256SUM"
+	rm -v "$RAMDISKDIR/GIT_BRANCH" "$RAMDISKDIR/GIT_HASH" "$RAMDISKDIR/SHA256SUM"
+	rm -rf "${TEMPDIR:?}/"
 	tar --append \
 		--file="$BACKUPFILE" \
-		--directory="$OPENWBBASEDIR/data/log/" \
+		--directory="$LOGDIR/" \
 		"backup.log"
 	echo "zipping archive"
 	gzip --verbose "$BACKUPFILE"

runs/factory_reset.sh

@@ -8,12 +8,28 @@ fi
 
 case "$1" in
 	"clearall")
+		echo "checking for configured cloud bridge..."
+		cloud_bridge=$(timeout 1 mosquitto_sub -t 'openWB/system/mqtt/bridge/+' | grep -E '"is_openwb_cloud": ?true')
+		if [ -n "$cloud_bridge" ]; then
+			echo "cloud bridge is configured"
+			valid_partner_ids=$(timeout 1 mosquitto_sub -t 'openWB/system/mqtt/valid_partner_ids' -C 1)
+		else
+			echo "no cloud bridge configured"
+		fi
 		echo "deleting retained message store of external mosquitto..."
 		timeout 3 mosquitto_sub -t '#' --remove-retained --retained-only
 		echo "deleting retained message store of internal mosquitto..."
 		timeout 3 mosquitto_sub -t '#' --remove-retained --retained-only -p 1886
 		echo "deleting log data"
 		rm -r "$OPENWBBASEDIR/data/charge_log/"* "$OPENWBBASEDIR/data/daily_log/"* "$OPENWBBASEDIR/data/log/"*.log "$OPENWBBASEDIR/data/monthly_log/"*
+		echo "reset display rotation"
+		sudo sed -i "s/^lcd_rotate=[0-3]$/lcd_rotate=0/" "/boot/config.txt"
+		if [ -n "$cloud_bridge" ]; then
+			echo "restore cloud bridge configuration: $cloud_bridge"
+			mosquitto_pub -t 'openWB/command/max_id/mqtt_bridge' -r -m 1 -p 1886
+			mosquitto_pub -t 'openWB/system/mqtt/valid_partner_ids' -r -m "$valid_partner_ids" -p 1886
+			mosquitto_pub -t 'openWB/system/mqtt/bridge/0' -r -m "$cloud_bridge" -p 1886
+		fi
 		echo "all done";;
 	*)
 		echo "please pass \"clearall\" as parameter if you really want to reset all data stored in the internal and external broker"

runs/prepare_restore.sh

@@ -15,11 +15,13 @@ declare resultStatus
 	mkdir -p "$WORKING_DIR"
 	echo "****************************************"
 	echo "Step 2: extract archive to working directory"
-	# extracting as root preserves file owner/group and permissions!
+	# extracting as root preserves file owner/group and permissions, but directory owner/group is root!
 	if ! sudo tar --verbose --extract --file="$SOURCE_FILE" --directory="$WORKING_DIR"; then
 		resultMessage="Beim Entpacken des Archivs ist ein Fehler aufgetreten!"
 		resultStatus=1
 	else
+		echo "fixing directory owner/group"
+		sudo chown --verbose openwb:openwb "$WORKING_DIR/openWB" "$WORKING_DIR/openWB/data" "$WORKING_DIR/openWB/data/log"
 		echo "****************************************"
 		echo "Step 3: validating extracted files"
 		if [[ ! -f "$WORKING_DIR/SHA256SUM" ]] ||
@@ -34,6 +36,12 @@ declare resultStatus
 			if [[ ! -f "$WORKING_DIR/configuration.json" ]]; then
 				echo "configuration missing; continue anyway"
 			fi
+			if [[ ! -d "$WORKING_DIR/conf_local.d" ]]; then
+				echo "mosquitto configuration missing; continue anyway"
+			fi
+			if [[ ! -d "$WORKING_DIR/boot" ]]; then
+				echo "boot file missing; continue anyway"
+			fi
 			if ! (cd "$WORKING_DIR" && sudo sha256sum --quiet --check "SHA256SUM"); then
 				resultMessage="Einige Dateien wurden gelöscht oder bearbeitet!"
 				resultStatus=1

runs/restore.sh

@@ -3,8 +3,9 @@ OPENWB_BASE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 RESTORE_DIR="$OPENWB_BASE_DIR/data/restore"
 SOURCE_FILE="$RESTORE_DIR/restore.tar.gz"
 WORKING_DIR="/home/openwb/openwb_restore"
-MOSQUITTO_DIR="/var/lib/mosquitto"
-MOSQUITTO_LOCAL_DIR="/var/lib/mosquitto_local"
+MOSQUITTO_DB_DIR="/var/lib/mosquitto"
+MOSQUITTO_LOCAL_DB_DIR="/var/lib/mosquitto_local"
+MOSQUITTO_CONF_DIR="/etc/mosquitto"
 LOG_FILE="$OPENWB_BASE_DIR/data/log/restore.log"
 
 {
@@ -54,19 +55,36 @@ LOG_FILE="$OPENWB_BASE_DIR/data/log/restore.log"
 		echo "Backup does not contain configuration. Skipping restore."
 	fi
 	echo "****************************************"
-	echo "Step 5: restore mosquitto db"
+	echo "Step 5.1: restore mosquitto db"
 	if [[ -f "${WORKING_DIR}/mosquitto/mosquitto.db" ]]; then
-		sudo mv -v -f "${WORKING_DIR}/mosquitto/mosquitto.db" "$MOSQUITTO_DIR/mosquitto.db"
+		sudo mv -v -f "${WORKING_DIR}/mosquitto/mosquitto.db" "$MOSQUITTO_DB_DIR/mosquitto.db"
 	else
 		echo "Backup does not contain mosquitto.db. Skipping restore."
 	fi
 	if [[ -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" ]]; then
-		sudo mv -v -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" "$MOSQUITTO_LOCAL_DIR/mosquitto.db"
+		sudo mv -v -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" "$MOSQUITTO_LOCAL_DB_DIR/mosquitto.db"
 	else
 		echo "Backup does not contain local mosquitto.db. Skipping restore."
 	fi
 	echo "****************************************"
-	echo "Step 6: cleanup after restore"
+	echo "Step 5.2: restore mosquitto configuration"
+	if [[ -d "${WORKING_DIR}/conf_local.d" ]]; then
+		# remove old configuration
+		sudo rm -v -r "$MOSQUITTO_CONF_DIR/conf_local.d"
+		# copy configuration
+		sudo cp -v -p -r "${WORKING_DIR}/conf_local.d" "$MOSQUITTO_CONF_DIR/"
+	else
+		echo "Backup does not contain mosquitto configuration. Skipping restore."
+	fi
+	echo "****************************************"
+	echo "Step 6: restore boot file"
+	if [[ -f "${WORKING_DIR}/boot/config.txt" ]]; then
+		sudo mv -v -f "${WORKING_DIR}/boot/config.txt" "/boot/"
+	else
+		echo "Backup does not contain boot file. Skipping restore."
+	fi
+	echo "****************************************"
+	echo "Step 7: cleanup after restore"
 	sudo rm "$SOURCE_FILE"
 	sudo rm -R "$WORKING_DIR"
 	echo "****************************************"