Merge pull request #377 from SimplyVC/release/1.3.0

Added version 1.3.0 code

Author: itsciccio

.dockerignore

@@ -1,7 +1,5 @@
 ./config
 ./alerter/logs
 ./certificates
-./web-installer/node_modules
-./web-installer/build
 ./ui/build
 ./ui/node_modules

.env

@@ -13,61 +13,65 @@ UI_ACCESS_IP=X.X.X.X
 # Development configuration
 DEV_MODE=false
 
-# Installer configuration
-INSTALLER_IP=172.18.0.3
-INSTALLER_PORT=8000
-INSTALLER_USERNAME=admin
-INSTALLER_PASSWORD=password
-ACCESS_TOKEN_SECRET=7dgtas97fgsd97tgs9tg3490t6g508th0hge80rhg0s90dgs
-ACCESS_TOKEN_LIFE=120
-REFRESH_TOKEN_SECRET=9g9g90df8ghd80gh80tg0h8rth085gh458h085h0g8y54hg
-REFRESH_TOKEN_LIFE=86400
-
 # Mongo configuration
 DB_NAME=panicdb
-DB_IP=172.18.0.2
-DB_IP_TEST=172.19.0.2
+
 DB_PORT=27017
-INSTALLER_AUTH_COLLECTION=installer_authentication
-ACCOUNTS_COLLECTION=accounts
+
+DB_IP_REPLICA_1=172.18.0.2
+DB_IP_REPLICA_1_TEST=172.19.0.2
+
+DB_IP_REPLICA_2=172.18.0.3
+DB_IP_REPLICA_2_TEST=172.19.0.3
+
+DB_IP_REPLICA_3=172.18.0.4
+DB_IP_REPLICA_3_TEST=172.19.0.4
+
+DB_IP_REPLICA_STARTUP=172.18.0.5
+DB_IP_REPLICA_STARTUP_TEST=172.19.0.5
 
 # Alerter configuration
-ALERTER_IP=172.18.0.4
+ALERTER_IP=172.18.0.7
 UNIQUE_ALERTER_IDENTIFIER=panic_alerter
 
 # Redis configuration
-REDIS_IP=172.18.0.5
-REDIS_IP_TEST=172.19.0.5
+REDIS_IP=172.18.0.8
+REDIS_IP_TEST=172.19.0.8
 REDIS_PORT=6379
 REDIS_DB=10
-REDIS_TEST_DB=11
+REDIS_DB_TEST=11
 
 # RabbitMQ configuration
-RABBIT_IP=172.18.0.6
-RABBIT_IP_TEST=172.19.0.6
+RABBIT_IP=172.18.0.9
+RABBIT_IP_TEST=172.19.0.9
 RABBIT_PORT=5672
 
 # Health Checker configuration
-HEALTH_CHECKER_IP=172.18.0.7
+HEALTH_CHECKER_IP=172.18.0.10
 
 # Tests configuration
-TESTS_IP=172.19.0.8
+TESTS_IP=172.19.0.11
 
 # UI configuration
-UI_DASHBOARD_IP=172.18.0.9
+UI_DASHBOARD_IP=172.18.0.12
 UI_DASHBOARD_PORT=3333
 
 # API configuration
-API_IP=172.18.0.10
+API_IP=172.18.0.13
+API_IP_TEST=172.19.0.13
 API_PORT=9000
+API_PORT_TEST=9001
 
 # Substrate API configuration
-SUBSTRATE_API_IP=172.18.0.11
+SUBSTRATE_API_IP=172.18.0.14
 SUBSTRATE_API_PORT=8080
 
+# Migration configuration
+MIGRATION_IP=172.18.0.15
+
 # Logs configuration - Log files with {} are Python template strings, where {}
 # is replaced with text that makes the log file name specific to the process
-# that logs to it. For example, system_monitor_{}.log may become 
+# that logs to it. For example, system_monitor_{}.log may become
 # system_monitor_validator.log.
 LOGGING_LEVEL=INFO
 DATA_STORE_LOG_FILE_TEMPLATE=logs/stores/{}.log
@@ -86,8 +90,8 @@ ALERTS_LOG_FILE=logs/alerts/alerts.log
 # becomes: https://api.github.com/repos/w3f/substrate/releases
 GITHUB_RELEASES_TEMPLATE=https://api.github.com/repos/{}releases
 
-# DockerHub monitoring configuration - This is a Python template string, where 
-# the two {} are replaced with the namespace and repository name, so that in the 
+# DockerHub monitoring configuration - This is a Python template string, where
+# the two {} are replaced with the namespace and repository name, so that in the
 # case of nginx which belongs to the namespace library, the complete link becomes
 # https://hub.docker.com/v2/repositories/library/nginx/tags
 DOCKERHUB_TAGS_TEMPLATE=https://hub.docker.com/v2/repositories/{}/{}/tags
@@ -101,8 +105,8 @@ NODE_MONITOR_PERIOD_SECONDS=10
 CHAINLINK_CONTRACTS_MONITOR_PERIOD_SECONDS=10
 NETWORK_MONITOR_PERIOD_SECONDS=60
 
-# Publishers limits - These define how much messages should be stored in a 
-# publisher queue before starting to prune old messages. This happens when for 
+# Publishers limits - These define how much messages should be stored in a
+# publisher queue before starting to prune old messages. This happens when for
 # some reason messages are not being sent by the publisher.
 DATA_TRANSFORMER_PUBLISHING_QUEUE_SIZE=1000
 ALERTER_PUBLISHING_QUEUE_SIZE=1000
@@ -118,4 +122,4 @@ ENABLE_LOG_ALERTS=True
 
 # Twilio Preferences
 TWIML=<Response><Reject/></Response>
-TWIML_IS_URL=false
+TWIML_IS_URL=false

.gitignore

@@ -8,10 +8,10 @@
 /cache
 
 # dependencies
-web-installer/node_modules
 ui/node_modules
 ui/www
 api/node_modules
+api/dist
 api/**/*.js
 # IDE
 .idea

.gitkeep

@@ -1,14 +1,13 @@
 # PANIC Monitoring and Alerting for Blockchains
 
-**DISCLAIMERS**: 
-- Don't allow public access to PANIC UI as it might contain sensitive information about your infrastructure. UI authentication is still to be developed.
+**DISCLAIMER**: Don't allow public access to PANIC UI as it might contain sensitive information about your infrastructure. UI authentication is still to be developed.
 
 ![PANIC Banner](./docs/images/PANIC_BANNER.png)
 
 PANIC is an open source monitoring and alerting solution for Cosmos-SDK, 
 Substrate and Chainlink based nodes by [Simply VC](https://simply-vc.com.mt/). 
-The tool was built with user friendliness in mind, and comes with numerous 
-features such as phone calls for critical alerts, a UI Dashboard, a Web-UI 
+The tool was built with user friendliness in mind and comes with numerous 
+features, such as phone calls for critical alerts, a UI Dashboard, a Web based 
 installation process and Telegram/Slack commands for increased control over your
 alerter.
 
@@ -82,19 +81,17 @@ git clone https://github.com/SimplyVC/panic
 cd panic
 ```
 
-Now that you're inside the PANIC directory, open up the .env file and change the `INSTALLER_USERNAME` and `INSTALLER_PASSWORD` fields to your preferred but secure choice. This is to ensure that when configuring PANIC through the web-installer no one else can access it. Moreover, the `UI_ACCESS_IP` field must also be changed to the IP of where PANIC UI is going to be hosted (can be set to `localhost` if running locally). This is to ensure that the API is only accessible from the UI. Helper scripts which can be used to get the IP address (`scripts/get_ip_linux.sh` and `scripts/get_ip_mac.sh`) are available but please note that these are not guaranteed to work on all servers/machines.
+Now that you're inside the PANIC directory, open up the .env file and change the `UI_ACCESS_IP` field to the IP of where PANIC UI is going to be hosted (can be set to `localhost` if running locally). This is to ensure that the API (PANIC UI Backend) is only accessible from the UI. Helper scripts which can be used to get the IP address (`scripts/get_ip_linux.sh` and `scripts/get_ip_mac.sh`) are available but please note that these are not guaranteed to work on all servers/machines.
 
 ```bash
 # This will access the .env file on your terminal
 nano .env
 ```
 
-Once inside change `UI_ACCESS_IP`, `INSTALLER_USERNAME` and `INSTALLER_PASSWORD` accordingly. Here is an example:
+Once inside change `UI_ACCESS_IP` accordingly. Here is an example:
 
 ```ini
 UI_ACCESS_IP=1.1.1.1
-INSTALLER_USERNAME=panic_operator
-INSTALLER_PASSWORD=wowthisisasecurepassword
 ```
 
 Then to exit hit the following keys:
@@ -118,13 +115,13 @@ docker system prune -a --volumes
 docker-compose up -d --build
 ```
 
-Now you will have to configure PANIC to monitor your nodes and systems as well as give it the channels to alert you through. To do this you will have to navigate to the running web-installer. This can be found at `https://{UI_ACCESS_IP}:8000`, or at `https://localhost:8000` if you're running it locally. The installer will first ask you to enter the username and password. These are `INSTALLER_USERNAME` and `INSTALLER_PASSWORD` which you have changed previously. Make sure you type **HTTPS** if you're getting an error when accessing the installer on your browser.
+The next step is to configure PANIC to monitor your nodes and systems as well as give it the channels to alert you through. You can do this by navigating to the PANIC UI at `https://{UI_ACCESS_IP}:3333`, or at `https://localhost:3333` if you're running it locally. The PANIC UI will start the installation procedure if it does not detect any configurations. Make sure you type **HTTPS** if you're getting an error when accessing PANIC UI on your browser.
 
-After you set-up PANIC to your liking, the Web-Installer will save these details in the form of configuration files inside the `config` folder. For correct behavior, these configuration files should never be modified. If you would like to edit them at some point, we suggest to re-run the web-installer again. The web-installer will ask you if you want to start a fresh install or load your old configuration.
+After you set-up PANIC to your liking the installation procedure will save these details in the Mongo database. For correct behavior the database should never be modified manually. If you would like to edit the configurations at some point you can do so by accessing the settings option on the PANIC UI header.
 
-PANIC will automatically read these configuration files and begin monitoring the data sources. To check that this is the case we suggest running the command `docker-compose logs -f alerter` and `docker-compose logs -f health-checker`. By this you can see the different components starting up. If you have set-up telegram/slack commands we suggest that you enter the command `/status` to check that all PANIC components are running. If you really want to check that PANIC is up and running, we suggest that you check that all the logs inside `panic/alerter/logs` have no errors.
+PANIC will automatically read these configuration files and begin monitoring the data sources. To confirm that PANIC is running as expected we suggest running the command `docker-compose logs -f alerter` and `docker-compose logs -f health-checker`. By this you can see the different components starting up. If you have set-up telegram/slack commands we suggest that you enter the command `/status` (telegram) or `/panicstatus` (slack) to check that all PANIC components are running. If you want to check that every PANIC component is up and running without any issue we suggest that you check that all the logs inside `panic/alerter/logs` have no errors.
 
-You can visualise node metrics and alerts using PANIC UI, which can be accessed at `https://{UI_ACCESS_IP}:3333`, or at `https://localhost:3333` if you're running it locally.
+After PANIC is up and running you can visualise node metrics and alerts using PANIC UI at `https://{UI_ACCESS_IP}:3333`, or at `https://localhost:3333` if you're running it locally.
 For more information regarding PANIC UI, [click here](./ui/README.md).
 
 Congratulations you should have PANIC up and running!
@@ -277,17 +274,11 @@ If you wish to explore more advanced features, PANIC also supports configurable
       3. The rest can be configured to your preferences.
       4. Click `Add Service`
     2. You will be taken to a new page, where you need to navigate to the `Integrations` tab and take note of the (i)`Integration Key`.
-  - Secondly the API Token
-    1. Navigate to `https:/{YOUR_SUBDOMAIN}.pagerduty.com/api_keys`.
-    2. Click `Create New API Key`
-    3. Give it a description (This can be anything you want)
-    4. Click `Create Key` and take note of (ii)`API Key` value.
 
 **At the end, you should have:**
 1. The Integration Key
-2. The API token
 
-These will be used later in the Web-Installer to be saved in the configuration files.
+This will be used later on in the installation procedure.
 
 **Note** You can also install an app for Android / iPhone as well as setup your phone number to receive alerts.
 
@@ -306,15 +297,15 @@ These will be used later in the Web-Installer to be saved in the configuration f
 **At the end, you should have:**
 1. The API token
 
-These will be used later in the Web-Installer to be saved in the configuration files.
+This will be used later on in the installation procedure.
 
 **Note** You can also install the Opsgenie app for Android / iPhone as well as setup your phone number to receive calls.
 
 [Back to Requirements](#requirements)
 
 ### Replacing SSL certificates (recommended)
 
-Apply your own SSL certificate signed by a certificate authority. The SSL certificate (cert.pem) and the key (key.pem) should be placed in the `panic/certificates` folder, and they should replace the existing dummy files. Note that these dummy files were given just for convenience as the Installer server won't start without them, however, for maximum security these must be replaced.
+Apply your own SSL certificate signed by a certificate authority. The SSL certificate (cert.pem) and the key (key.pem) should be placed in the `panic/certificates` folder, and they should replace the existing dummy files. Note that these dummy files were given just for convenience as the API (PANIC UI Backend) server won't start without them, however, for maximum security these must be replaced.
 
 We suggest reading [this](https://nodejs.org/en/knowledge/HTTP/servers/how-to-create-a-HTTPS-server/) for more details on SSL certificates, and how to generate a self signed certificate in case you do not want to obtain a certificate signed by a certificate authority. However, for maximum security, the self signed certificate option is not recommended.
 
@@ -334,6 +325,13 @@ npm install         # Install API project dependencies
 npm test            # Run API unit tests
 ```
 
+To run the tests for the **Substrate API** component within PANIC, navigate to the 
+`/substrate-api` directory and do the following:
+```bash
+npm install         # Install API project dependencies
+npm test            # Run API unit tests
+```
+
 To run the tests for the **UI** component within PANIC, navigate to the
 `/ui` directory and do the following:
 ```bash

README.md

@@ -18,7 +18,7 @@
 from src.alerter.managers.substrate import SubstrateAlertersManager
 from src.alerter.managers.system import SystemAlertersManager
 from src.channels_manager.manager import ChannelsManager
-from src.config_manager import ConfigsManager
+from src.config_manager.change_stream.config_manager import ConfigsManager
 from src.data_store.stores.manager import StoreManager
 from src.data_transformers.manager import DataTransformersManager
 from src.message_broker.rabbitmq import RabbitMQApi
@@ -30,7 +30,6 @@
 from src.monitors.managers.node import NodeMonitorsManager
 from src.monitors.managers.system import SystemMonitorsManager
 from src.utils import env
-from src.utils.constants.configs import IGNORE_FILE_PATTERNS
 from src.utils.constants.names import (
     SYSTEM_ALERTERS_MANAGER_NAME, GITHUB_ALERTER_MANAGER_NAME,
     DOCKERHUB_ALERTER_MANAGER_NAME, SYSTEM_MONITORS_MANAGER_NAME,
@@ -589,13 +588,12 @@ def _initialise_config_manager() -> Tuple[ConfigsManager, logging.Logger]:
     config_manager_logger = _initialise_logger(
         display_name, ConfigsManager.__name__, env.CONFIG_MANAGER_LOG_FILE
     )
-
+    
     rabbit_ip = env.RABBIT_IP
     while True:
         try:
             config_manager = ConfigsManager(
-                display_name, config_manager_logger, '../config', rabbit_ip,
-                ignore_file_patterns=IGNORE_FILE_PATTERNS
+                display_name, config_manager_logger, rabbit_ip                
             )
             return config_manager, config_manager_logger
         except Exception as e:

alerter/run_alerter.py

@@ -39,6 +39,7 @@
 from src.data_store.redis import RedisApi
 from src.message_broker.rabbitmq import RabbitMQApi
 from src.utils import env
+from src.utils.constants.mongo import REPLICA_SET_HOSTS, REPLICA_SET_NAME
 from src.utils.constants.names import (TELEGRAM_ALERTS_HANDLER_NAME_TEMPLATE,
                                        TELEGRAM_COMMANDS_HANDLER_NAME_TEMPLATE,
                                        TELEGRAM_COMMAND_HANDLERS_NAME,
@@ -174,7 +175,8 @@ def _initialise_telegram_commands_handler(
                 namespace=env.UNIQUE_ALERTER_IDENTIFIER)
             cmd_handlers_mongo = MongoApi(
                 logger=cmd_handlers_logger.getChild(MongoApi.__name__),
-                host=env.DB_IP, db_name=env.DB_NAME, port=env.DB_PORT)
+                host=REPLICA_SET_HOSTS, db_name=env.DB_NAME,
+                replicaSet=REPLICA_SET_NAME)
 
             cmd_handlers = TelegramCommandHandlers(
                 TELEGRAM_COMMAND_HANDLERS_NAME, cmd_handlers_logger,
@@ -282,7 +284,8 @@ def _initialise_slack_commands_handler(
                 namespace=env.UNIQUE_ALERTER_IDENTIFIER)
             cmd_handlers_mongo = MongoApi(
                 logger=cmd_handlers_logger.getChild(MongoApi.__name__),
-                host=env.DB_IP, db_name=env.DB_NAME, port=env.DB_PORT)
+                host=REPLICA_SET_HOSTS, db_name=env.DB_NAME,
+                replicaSet=REPLICA_SET_NAME)
 
             cmd_handlers = SlackCommandHandlers(
                 SLACK_COMMAND_HANDLERS_NAME, cmd_handlers_logger,
@@ -597,4 +600,4 @@ def start_handler(handler: ChannelHandler) -> None:
             log_and_print(get_stopped_message(handler), handler.logger)
             log_and_print("Restarting {} in {} seconds.".format(
                 handler, RESTART_SLEEPING_PERIOD), handler.logger)
-            time.sleep(RESTART_SLEEPING_PERIOD)
+            time.sleep(RESTART_SLEEPING_PERIOD)