Add NetFlow collector and Grafolean NetFlow bot

Author: grafolean

.env.example

@@ -0,0 +1,4 @@
+BACKEND_URL=https://grafolean.com/api
+BOT_TOKEN=
+JOBS_REFRESH_INTERVAL=60
+NETFLOW_PORT=2055

.gitignore

@@ -1 +1,2 @@
 .vscode
+.env

Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.6-slim-stretch as python-requirements
+COPY ./Pipfile ./Pipfile.lock /netflowbot/
+WORKDIR /netflowbot
+RUN \
+    pip install pipenv && \
+    pipenv lock -r > /requirements.txt
+
+FROM python:3.6-slim-stretch as build-backend
+COPY ./ /netflowbot/
+WORKDIR /netflowbot
+RUN \
+    find ./ ! -name '*.py' -type f -exec rm '{}' ';' && \
+    rm -rf tests/ .vscode/ .pytest_cache/ __pycache__/ && \
+    python3.6 -m compileall -b ./ && \
+    find ./ -name '*.py' -exec rm '{}' ';'
+
+
+FROM python:3.6-slim-stretch
+ARG VERSION
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL org.label-schema.vendor="Grafolean" \
+      org.label-schema.url="https://grafolean.com/" \
+      org.label-schema.name="Grafolean NetFlow bot" \
+      org.label-schema.description="NetFlow collector and Grafolean NetFlow bot" \
+      org.label-schema.version=$VERSION \
+      org.label-schema.vcs-url="https://github.com/grafolean/grafolean-netflow-bot/" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.docker.schema-version="1.0"
+COPY --from=python-requirements /requirements.txt /requirements.txt
+RUN \
+    pip install --no-cache-dir -r /requirements.txt && \
+    echo "alias l='ls -altr'" >> /root/.bashrc
+COPY --from=build-backend /netflowbot/ /netflowbot/
+WORKDIR /netflowbot
+# check for "fail" file and if it exists, remove it and fail the check:
+HEALTHCHECK --interval=10s --retries=1 CMD /bin/bash -c "[ ! -f /tmp/fail_health_check ] || ( rm /tmp/fail_health_check && exit 1 )"
+
+# CAREFUL:
+# There are two entrypoints, both of which should be running:
+# - netflowcollector: gathering packets and writing statistics to Redis
+# - netflowbot: Grafolean bot for NetFlow - sending data to Grafolean according to configured sensors
+#CMD ["python", "-m", "netflowcollector"]
+CMD ["python", "-m", "netflowbot"]

Pipfile

@@ -10,6 +10,8 @@ pylint = "*"
 requests = "*"
 python-dotenv = "*"
 ansicolors = "*"
+grafoleancollector = "*"
+redis = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -0,0 +1,12 @@
+version: '2.1'
+services:
+
+  # Build a local Docker image. Usage:
+  #   $ docker-compose -f docker-compose.dev.yml build
+
+  grafolean-netflow-bot:
+    image: grafolean/grafolean-netflow-bot
+    container_name: grafolean-netflow-bot
+    build:
+      context: .
+      dockerfile: Dockerfile

docker-compose.dev.yml

@@ -0,0 +1,77 @@
+version: '2.1'
+services:
+
+  #
+  # Welcome to Grafolean NetFlow bot!
+  #
+  # This file should be modified to suit your running environment. Please check the comments and change
+  # the settings appropriately.
+  #
+
+  netflowbot:
+    # If you wish to load an explicit version, change the next line. For example:
+    #   image: grafolean/grafolean-netflow-bot:v1.0.0
+    image: grafolean/grafolean-netflow-bot
+    container_name: grafolean-netflow-bot
+    environment:
+      # Backend url must be set to the address of the Grafolean backend, for example this uses Grafolean hosted service:
+      # - BACKEND_URL=https://grafolean.com/api
+      # IMPORTANT: '127.0.0.1' and 'localhost' are _never_ correct addresses for Grafolean backend, because they translate
+      # to container, not host.
+      - BACKEND_URL=${BACKEND_URL}
+      # To use NetFlow bot, a bot with the protocol "netflow" must be added via user interface, then the token needs to be copied here:
+      - BOT_TOKEN=${BOT_TOKEN}
+      # Interval between fetching information about jobs:
+      - JOBS_REFRESH_INTERVAL=${JOBS_REFRESH_INTERVAL:-60}
+      - REDIS_HOST=redis
+    restart: always
+    networks:
+      - grafolean
+
+
+  redis:
+    image: redis:5-alpine
+    container_name: grafolean-netflow-bot-redis
+    restart: always
+    networks:
+      - grafolean
+
+
+  netflowcollector:
+    # This process collects NetFlow data:
+    image: grafolean/grafolean-netflow-bot
+    container_name: grafolean-netflow-collector
+    environment:
+      - REDIS_HOST=redis
+      - NETFLOW_PORT=2055
+    ports:
+      - "${NETFLOW_PORT:-2055}:2055/udp"
+    restart: always
+    # CAREFUL: NetFlow collector uses the same docker image as bot
+    # (grafolean/grafolean-netflow-bot), but specifies a different entrypoint:
+    entrypoint:
+      - python
+      - -m
+      - netflowcollector
+    networks:
+      - grafolean
+
+
+  autoheal:
+    # This container automatically restarts any container that fails its health check. Not a bullet-proof solution, but better than nothing.
+    image: willfarrell/autoheal
+    container_name: grafolean-netflow-bot-autoheal
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: always
+    networks:
+      - grafolean
+
+
+# If running on the same host, join the Grafolean network, so we can reach Grafolean
+# backend at address grafolean:80.
+networks:
+  grafolean:
+    name: grafolean