BE: SYNC API logging

docs/SECURITY.md

@@ -8,7 +8,7 @@ This includes (but is not limited to):
 - Running NetAlertX only on networks where you have legal authorization
 - Keeping your deployment up to date with the latest patches
 
-> NetAlertX is not responsible for misuse, misconfiguration, or unsecure deployments. Always test and secure your setup before exposing it to the outside world.
+> NetAlertX is not responsible for misuse, misconfiguration, or insecure deployments. Always test and secure your setup before exposing it to the outside world. Users interacting with the UI are treated as trusted actors within the deployment model.
 
 # 🔐 Securing Your NetAlertX Instance
 
@@ -36,7 +36,7 @@ NetAlertX is designed to be run on **private LANs**, not the open internet.
 
 ### ✅ Tailscale (Easy VPN Alternative)
 
-Tailscale sets up a private mesh network between your devices. It's fast to configure and ideal for NetAlertX.  
+Tailscale sets up a private mesh network between your devices. It's fast to configure and ideal for NetAlertX.
 👉 [Get started with Tailscale](https://tailscale.com/)
 
 ---
@@ -63,19 +63,19 @@ By default, NetAlertX does **not** require login. Before exposing the UI in any
 
 ## 🔥 Additional Security Measures
 
-- **Firewall / Network Rules**  
+- **Firewall / Network Rules**
   Restrict UI/API access to trusted IPs only.
 
-- **Limit Docker Capabilities**  
+- **Limit Docker Capabilities**
   Avoid `--privileged`. Use `--cap-add=NET_RAW` and others **only if required** by your scan method.
 
-- **Keep NetAlertX Updated**  
+- **Keep NetAlertX Updated**
   Regular updates contain bug fixes and security patches.
 
-- **Plugin Permissions**  
+- **Plugin Permissions**
   Disable unused plugins. Only install from trusted sources.
 
-- **Use Read-Only API Keys**  
+- **Use Read-Only API Keys**
   When integrating NetAlertX with other tools, scope keys tightly.
 
 ---

front/plugins/sync/sync.py

@@ -182,13 +182,29 @@ def main():
                 # are pipe-delimited — catch and skip them via the JSONDecodeError guard below.
                 parts = file_name.split('.')
                 if len(parts) > 2:
-                    # Extract node name:
-                    #   decoded/encoded: last_result.PLUGIN.decoded.NodeName.N.log → parts[3]
-                    #   pull mode:       last_result.NodeName.log                  → parts[1]
-                    if 'decoded' in file_name or 'encoded' in file_name:
-                        syncHubNodeName = parts[3]
+                    # PUSH artifacts:
+                    #   last_result.PLUGIN.decoded.NodeName.N.log
+                    #   last_result.PLUGIN.encoded.NodeName.N.log
+                    #
+                    # Require BOTH:
+                    #   1. decoded/encoded marker
+                    #   2. trailing ".<counter>.log" shape
+                    #
+                    # This prevents PULL filenames like:
+                    #   last_result.office.encoded.lab.log
+                    # from being incorrectly parsed as PUSH artifacts.
+                    is_push_artifact = (
+                        ('.decoded.' in file_name or '.encoded.' in file_name) and file_name.rsplit('.', 2)[1].isdigit()
+                    )
+
+                    if is_push_artifact:
+                        _marker = '.decoded.' if '.decoded.' in file_name else '.encoded.'
+                        _, _after = file_name.split(_marker, 1)
+                        syncHubNodeName = _after.rsplit('.', 2)[0]
                     else:
-                        syncHubNodeName = parts[1]
+                        # PULL artifact:
+                        #   last_result.NodeName.log
+                        syncHubNodeName = file_name[len('last_result.'):-len('.log')]
 
                     file_path = f"{LOG_PATH}/{file_name}"
 
@@ -284,7 +300,6 @@ def main():
     return 0
 
 
-# ------------------------------------------------------------------
 # Data retrieval methods
 api_endpoints = [
     "/sync",  # New Python-based endpoint
@@ -293,39 +308,87 @@ def main():
 
 # send data to the HUB
 def send_data(api_token, file_content, encryption_key, file_path, node_name, pref, hub_url):
-    """Send encrypted data to HUB, preferring /sync endpoint and falling back to PHP version."""
+    """
+    Sends encrypted plugin output from NODE → HUB.
+
+    Flow:
+    1. Encrypt plugin output locally
+    2. Build payload (data + metadata)
+    3. Try each configured HUB endpoint in order
+    4. On success (200) → stop immediately
+    5. On failure → log HUB response + continue fallback
+    6. If all endpoints fail → alert user
+    """
+
+    # STEP 1: Encrypt raw plugin output before transmission
     encrypted_data = encrypt_data(file_content, encryption_key)
-    mylog('verbose', [f'[{pluginName}] Sending encrypted_data: "{encrypted_data}"'])
 
+    mylog('verbose', [f"[{pluginName}] Encrypted payload prepared type={type(encrypted_data).__name__}"])
+
+    # STEP 2: Build request payload for HUB sync API
     data = {
         'data': encrypted_data,
         'file_path': file_path,
         'plugin': pref,
         'node_name': node_name
     }
-    headers = {'Authorization': f'Bearer {api_token}'}
 
+    headers = {
+        'Authorization': f'Bearer {api_token}'
+    }
+
+    # STEP 3: Attempt delivery to each configured endpoint
     for endpoint in api_endpoints:
 
         final_endpoint = hub_url + endpoint
 
         try:
-            response = requests.post(final_endpoint, json=data, headers=headers, timeout=5)
-            mylog('verbose', [f'[{pluginName}] Tried endpoint: {final_endpoint}, status: {response.status_code}'])
 
+            # STEP 4: Send request to HUB sync endpoint
+            response = requests.post(
+                final_endpoint,
+                json=data,
+                headers=headers,
+                timeout=5
+            )
+
+            # STEP 5a: Success path (HUB accepted payload)
             if response.status_code == 200:
-                message = f'[{pluginName}] Data for "{file_path}" sent successfully via {final_endpoint}'
+                message = (f'[{pluginName}] Sync success for "{file_path}" via {final_endpoint}')
                 mylog('verbose', [message])
                 write_notification(message, 'info', timeNowUTC())
                 return True
 
+            # STEP 5b: HUB returned error (e.g. 500, 400)
+            try:
+                response_json = response.json()
+            except Exception:
+                response_json = {}
+
+            # Extract best available error message
+            error_msg = (
+                response_json.get("error") or response_json.get("message") or response.text
+            )
+
+            msg = (f'[{pluginName}] HUB error on {final_endpoint} [{response.status_code}]: {error_msg}')
+
+            mylog('none', [msg])
+            write_notification(msg, 'alert', timeNowUTC())
+
+            mylog('verbose', [f'[{pluginName}] Endpoint attempted: {final_endpoint} status={response.status_code}'])
+
         except requests.RequestException as e:
-            mylog('verbose', [f'[{pluginName}] Error calling {final_endpoint}: {e}'])
+            # STEP 5c: Network-level failure (timeout, DNS, etc.)
+            mylog('verbose', [f'[{pluginName}] Request exception calling {final_endpoint} error={type(e).__name__}: {e}'])
 
-    # If all endpoints fail
-    message = f'[{pluginName}] Failed to send data for "{file_path}" via all endpoints'
-    mylog('verbose', [message])
+    # STEP 6: All endpoints failed → final fallback alert
+    message = (
+        f'[{pluginName}] All HUB endpoints failed for "{file_path}"'
+    )
+
+    mylog('none', [message])
     write_notification(message, 'alert', timeNowUTC())
+
     return False
 
 

server/api_server/sync_endpoint.py

@@ -1,3 +1,4 @@
+import json
 import os
 import base64
 from flask import jsonify, request
@@ -6,16 +7,14 @@
 from utils.datetime_utils import timeNowUTC
 from messaging.in_app import write_notification
 
-INSTALL_PATH = os.getenv("NETALERTX_APP", "/app")
-
 # Make sure log level is initialized correctly
 lggr = Logger(get_setting_value('LOG_LEVEL'))
 
 
 def handle_sync_get():
     """Handle GET requests for SYNC (NODE → HUB)."""
 
-    # get all dwevices from the api endpoint
+    # get all devices from the api endpoint
     api_path = os.environ.get('NETALERTX_API', '/tmp/api')
 
     file_path = f"/{api_path}/table_devices.json"
@@ -47,40 +46,100 @@ def handle_sync_get():
 
 def handle_sync_post():
     """Handle POST requests for SYNC (HUB receiving from NODE)."""
-    body = request.get_json(silent=True) or {}
+
+    mylog("debug", [
+        "[SYNC API] ENTER handle_sync_post",
+        f"method={request.method}",
+        f"content_type={request.content_type}",
+        f"content_length={request.content_length}",
+        f"remote_addr={request.remote_addr}"
+    ])
+
+    # ---- RAW BODY (critical for debugging encoding / encryption issues)
+    try:
+        raw = request.get_data(cache=False)
+        mylog("debug", [
+            f"[SYNC API] raw_bytes_len={len(raw)} raw_preview={raw[:200]}"
+        ])
+    except Exception as e:
+        mylog("none", [f"[SYNC API] FAILED reading raw body: {e}"])
+        write_notification("[SYNC API] FAILED reading raw body - see app.log", 'alert', timeNowUTC())
+        return jsonify({"error": "failed reading body"}), 400
+
+    # ---- JSON PARSE (from already-read raw bytes to avoid empty-stream re-read)
+    try:
+        body = json.loads(raw)
+        mylog("debug", [f"[SYNC API] parsed_json={body}"])
+    except Exception as e:
+        msg = f"[SYNC API] JSON_PARSE_FAILED={e}"
+        mylog("none", [msg])
+        write_notification(msg, 'alert', timeNowUTC())
+        return jsonify({"error": "invalid json"}), 400
+
+    # ---- EXTRACT FIELDS
     data = body.get("data", "")
     node_name = body.get("node_name", "")
     plugin = body.get("plugin", "")
 
-    storage_path = INSTALL_PATH + "/log/plugins"
-    os.makedirs(storage_path, exist_ok=True)
-
-    encoded_files = [
-        f
-        for f in os.listdir(storage_path)
-        if f.startswith(f"last_result.{plugin}.encoded.{node_name}")
-    ]
-    decoded_files = [
-        f
-        for f in os.listdir(storage_path)
-        if f.startswith(f"last_result.{plugin}.decoded.{node_name}")
-    ]
-    file_count = len(encoded_files + decoded_files) + 1
+    mylog("debug", [
+        f"[SYNC API] node_name={repr(node_name)} plugin={repr(plugin)} data_type={type(data).__name__} data_len={len(data) if isinstance(data, str) else 'non-string'}"
+    ])
+
+    storage_path = os.getenv("NETALERTX_PLUGINS_LOG", "/tmp/log/plugins")
+
+    try:
+        os.makedirs(storage_path, exist_ok=True)
+        mylog("debug", [f"[SYNC API] storage_path_ready={storage_path}"])
+    except Exception as e:
+        msg = f"[SYNC API] MKDIR_FAILED={e}"
+        mylog("none", [msg])
+        write_notification(msg, 'alert', timeNowUTC())
+        return jsonify({"error": "storage path error"}), 500
 
+    # ---- FILE COUNT LOGIC
+    try:
+        encoded_files = [
+            f for f in os.listdir(storage_path)
+            if f.startswith(f"last_result.{plugin}.encoded.{node_name}")
+        ]
+        decoded_files = [
+            f for f in os.listdir(storage_path)
+            if f.startswith(f"last_result.{plugin}.decoded.{node_name}")
+        ]
+        file_count = len(encoded_files + decoded_files) + 1
+
+        mylog("debug", [f"[SYNC API] encoded_files={len(encoded_files)} decoded_files={len(decoded_files)} file_count={file_count}"])
+    except Exception as e:
+        msg = f"[SYNC API] LISTDIR_FAILED={e}"
+        mylog("none", [msg])
+        write_notification(msg, 'alert', timeNowUTC())
+        return jsonify({"error": "listdir failed"}), 500
+
+    # ---- FILE PATH
     file_path_new = os.path.join(
-        storage_path, f"last_result.{plugin}.encoded.{node_name}.{file_count}.log"
+        storage_path,
+        f"last_result.{plugin}.encoded.{node_name}.{file_count}.log"
     )
 
+    mylog("verbose", [f"[SYNC API] file_path_new={file_path_new}"])
+
     try:
+        if not isinstance(data, str):
+            data = str(data)
+
         with open(file_path_new, "w") as f:
             f.write(data)
+
     except Exception as e:
-        msg = f"[Plugin: SYNC] Failed to store data: {e}"
-        write_notification(msg, "alert", timeNowUTC())
-        mylog("verbose", [msg])
-        return jsonify({"error": msg}), 500
+
+        msg = f"[Plugin: SYNC] Data write failed ({file_path_new}): {e}"
+        mylog("none", [msg])
+        write_notification(msg, 'alert', timeNowUTC())
+        return jsonify({"error": str(e)}), 500
 
     msg = f"[Plugin: SYNC] Data received ({file_path_new})"
-    write_notification(msg, "info", timeNowUTC())
+    if lggr.isAbove('verbose'):
+        write_notification(msg, 'info', timeNowUTC())
     mylog("verbose", [msg])
+
     return jsonify({"message": "Data received and stored successfully"}), 200

server/plugin.py

@@ -824,8 +824,7 @@ def process_plugin_events(db, plugin, plugEventsArr):
                         # find corresponding object for the event and merge
                         if plugObj.idsHash == tmpObjFromEvent.idsHash:
                             if (
-                                plugObj.status == "missing-in-last-scan"
-                                or tmpObjFromEvent.status == "watched-changed"
+                                plugObj.status == "missing-in-last-scan" or tmpObjFromEvent.status == "watched-changed"
                             ):
                                 changed_this_cycle.add(tmpObjFromEvent.idsHash)
                             pluginObjects[index] = combine_plugin_objects(

server/utils/datetime_utils.py

@@ -6,7 +6,7 @@
 import re
 import pytz
 from typing import Union, Optional
-from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+from zoneinfo import ZoneInfo
 import email.utils
 import conf
 # from const import *

test/plugins/test_sync_protocol.py

@@ -71,11 +71,18 @@ def _node_name_from_filename(file_name: str) -> str:
     """Mirror of the node-name extraction in sync.main() (Mode 3).
 
     Real file formats produced by the system:
-      PUSH (post-decode): last_result.PLUGIN.decoded.NodeName.N.log  → parts[3]
-      PULL:               last_result.NodeName.log                   → parts[1]
+      PUSH (post-decode): last_result.PLUGIN.decoded.NodeName.N.log
+        — split on '.decoded.' marker, strip .N.log with rsplit from the right
+      PULL:               last_result.NodeName.log
+        — strip 'last_result.' prefix and '.log' suffix
+
+    Both forms handle dots anywhere in PLUGIN or NodeName.
     """
-    parts = file_name.split(".")
-    return parts[3] if ("decoded" in file_name or "encoded" in file_name) else parts[1]
+    if '.decoded.' in file_name or '.encoded.' in file_name:
+        marker = '.decoded.' if '.decoded.' in file_name else '.encoded.'
+        _, after = file_name.split(marker, 1)
+        return after.rsplit('.', 2)[0]
+    return file_name[len('last_result.'):-len('.log')]
 
 
 def _should_delete_after_process(filename: str) -> bool:
@@ -339,6 +346,26 @@ def test_push_decoded_different_plugins(self):
             assert _node_name_from_filename(fname) == "HubNode", \
                 f"Expected 'HubNode' from {fname}"
 
+    # --- dot-in-identifier regression (fragile parts[3] fix) ---
+
+    def test_pull_node_name_with_dots(self):
+        # PULL mode: node name set to e.g. "node.home" or an IP like "192.168.1.82"
+        assert _node_name_from_filename("last_result.node.home.log") == "node.home"
+        assert _node_name_from_filename("last_result.192.168.1.82.log") == "192.168.1.82"
+
+    def test_push_decoded_node_name_with_dots(self):
+        # Node name "Node.Vlan01" must survive the filename round-trip intact
+        assert _node_name_from_filename("last_result.ARPSCAN.decoded.Node.Vlan01.1.log") == "Node.Vlan01"
+
+    def test_push_decoded_plugin_name_with_dots(self):
+        # Hypothetical plugin with a dot in its name must not shift the node index
+        assert _node_name_from_filename("last_result.MY.PLUGIN.decoded.NodeA.1.log") == "NodeA"
+
+    def test_push_both_identifiers_with_dots(self):
+        assert _node_name_from_filename(
+            "last_result.A.B.decoded.x.y.z.1.log"
+        ) == "x.y.z"
+
 
 # ===========================================================================
 # CurrentScan candidates filter (Mode 3 – RECEIVE)