Feature pii

api/ingestor/usage.ts

@@ -901,7 +901,6 @@ const usage: IngestorUsageModule = {
         if (Object.keys(update).length > 0) {
             ob.updates.push(update);
         }
-        usage.processCoreMetrics(params); // Collects core metrics
     },
 
     /**

api/parts/mgmt/app_users.js

@@ -1230,6 +1230,7 @@ usersApi.export = function(app_id, query, params, callback) {
                         export_commands: export_commands,
                         query: query,
                         uids: res[0].uid,
+                        export_id: export_id,
                         export_folder: export_folder
                     }, function() {
                         var commands = [];

api/utils/requestProcessor.js

@@ -1614,25 +1614,35 @@ const processRequest = (params) => {
                                         eid = eid[0];
 
                                         var cursor = common.db.collection("exports").find({"_eid": eid}, {"_eid": 0, "_id": 0});
-                                        var options = {"type": "stream", "filename": eid + ".json", params: params};
                                         params.res.writeHead(200, {
-                                            'Content-Type': 'application/x-gzip',
-                                            'Content-Disposition': 'inline; filename="' + eid + '.json'
+                                            'Content-Type': 'application/json',
+                                            'Content-Disposition': 'inline; filename="' + eid + '.json"'
                                         });
-                                        options.streamOptions = {};
-                                        if (options.type === "stream" || options.type === "json") {
-                                            options.streamOptions.transform = function(doc) {
+
+                                        var isFirst = true;
+                                        params.res.write('[');
+                                        cursor.forEach(function(doc) {
+                                            if (doc) {
                                                 doc._id = doc.__id;
                                                 delete doc.__id;
-                                                return JSON.stringify(doc);
-                                            };
-                                        }
-
-                                        options.output = options.output || function(stream) {
-                                            countlyApi.data.exports.stream(options.params, stream, options);
-                                        };
-                                        options.output(cursor);
-
+                                                if (!isFirst) {
+                                                    params.res.write(',');
+                                                }
+                                                isFirst = false;
+                                                params.res.write(JSON.stringify(doc));
+                                            }
+                                        }).then(function() {
+                                            params.res.write(']');
+                                            params.res.end();
+                                        }).catch(function(err) {
+                                            log.e('Error streaming export data:', err);
+                                            if (!params.res.headersSent) {
+                                                common.returnMessage(params, 500, 'Error streaming export data');
+                                            }
+                                            else {
+                                                params.res.end();
+                                            }
+                                        });
 
                                     }
                                     else {

plugins/alerts/api/alertModules/pii.js

@@ -0,0 +1,153 @@
+/**
+ * @typedef {import('../parts/common-lib.js').App} App
+ */
+
+const log = require('../../../../api/utils/log.js')('alert:pii');
+const moment = require('moment-timezone');
+const common = require('../../../../api/utils/common.js');
+const commonLib = require("../parts/common-lib.js");
+const { ObjectId } = require('mongodb');
+
+module.exports.triggerByEvent = triggerByEvent;
+/**
+ * Checks if given payload contains PII incidents and triggers matching alerts.
+ * @param {object} payload - { incidents, app }
+ */
+async function triggerByEvent(payload) {
+    const incidents = payload?.incidents;
+    const app = payload?.app;
+
+    if (!incidents || !Array.isArray(incidents) || !incidents.length || !app) {
+        return;
+    }
+
+    // Find alerts for this specific app AND alerts targeting all apps
+    const [appAlerts, allAppsAlerts] = await Promise.all([
+        common.readBatcher.getMany("alerts", {
+            selectedApps: app._id.toString(),
+            alertDataType: "pii",
+            alertDataSubType: commonLib.TRIGGERED_BY_EVENT.pii,
+            enabled: true,
+        }),
+        common.readBatcher.getMany("alerts", {
+            selectedApps: "all",
+            alertDataType: "pii",
+            alertDataSubType: commonLib.TRIGGERED_BY_EVENT.pii,
+            enabled: true,
+        }),
+    ]);
+
+    const alerts = [...(appAlerts || []), ...(allAppsAlerts || [])];
+    if (!alerts.length) {
+        return;
+    }
+
+    await Promise.all(alerts.map(alert => {
+        // If alert targets a specific rule, only trigger for incidents matching that rule
+        if (alert.alertDataSubType2) {
+            const matchingIncidents = incidents.filter(inc => inc.ruleId === alert.alertDataSubType2);
+            if (!matchingIncidents.length) {
+                return Promise.resolve();
+            }
+            return commonLib.trigger({
+                alert,
+                app,
+                date: new Date(),
+                extra: {
+                    incidentCount: matchingIncidents.length,
+                    firstIncident: matchingIncidents[0],
+                }
+            }, log);
+        }
+        return commonLib.trigger({
+            alert,
+            app,
+            date: new Date(),
+            extra: {
+                incidentCount: incidents.length,
+                firstIncident: incidents[0],
+            }
+        }, log);
+    }));
+}
+
+
+module.exports.check = async function({ alertConfigs: alert, scheduledTo: date }) {
+    const selectedApp = alert.selectedApps[0];
+    let app = null;
+    let appId = null;
+
+    if (selectedApp !== "all") {
+        app = await common.readBatcher.getOne("apps", { _id: new ObjectId(selectedApp) });
+        if (!app) {
+            log.e(`App ${selectedApp} couldn't be found`);
+            return;
+        }
+        appId = app._id.toString();
+    }
+
+    let { period, compareType, compareValue, filterValue, alertDataSubType2 } = alert;
+    compareValue = Number(compareValue);
+
+    const metricValue = await countIncidents(date, period, appId, filterValue, alertDataSubType2) || 0;
+
+    if (compareType === commonLib.COMPARE_TYPE_ENUM.MORE_THAN) {
+        if (metricValue > compareValue) {
+            await commonLib.trigger({ alert, app, metricValue, date }, log);
+        }
+    }
+    else {
+        const before = moment(date).subtract(1, commonLib.PERIOD_TO_DATE_COMPONENT_MAP[period]).toDate();
+        const metricValueBefore = await countIncidents(before, period, appId, filterValue, alertDataSubType2);
+        if (!metricValueBefore) {
+            return;
+        }
+
+        const change = (metricValue / metricValueBefore - 1) * 100;
+        const shouldTrigger = compareType === commonLib.COMPARE_TYPE_ENUM.INCREASED_BY
+            ? change >= compareValue
+            : change <= -compareValue;
+
+        if (shouldTrigger) {
+            await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore }, log);
+        }
+    }
+};
+
+/**
+ * Count PII incidents within a time period.
+ * @param   {Date}                      date         - end date of the period
+ * @param   {string}                    period       - hourly|daily|monthly
+ * @param   {string|null}               appId        - app ID to filter by, or null for all apps
+ * @param   {string|undefined}          actionFilter - optional action filter (NOTIFY|OBFUSCATE|BLOCK)
+ * @param   {string|undefined}          ruleId       - optional PII rule ID to filter by
+ * @returns {Promise<number>}                        - incident count
+ */
+async function countIncidents(date, period, appId, actionFilter, ruleId) {
+    const periodMs = {
+        hourly: 60 * 60 * 1000,
+        daily: 24 * 60 * 60 * 1000,
+        monthly: 30 * 24 * 60 * 60 * 1000,
+    };
+
+    const endTs = date.getTime();
+    const startTs = endTs - (periodMs[period] || periodMs.daily);
+
+    const query = {
+        ts: { $gte: startTs, $lte: endTs },
+    };
+
+    if (appId) {
+        query.app_id = appId;
+    }
+
+    if (actionFilter) {
+        query.action = actionFilter;
+    }
+
+    if (ruleId) {
+        query.ruleId = ruleId;
+    }
+
+    return common.db.collection("pii_incidents").countDocuments(query);
+}

plugins/alerts/api/ingestor.js

@@ -20,7 +20,7 @@ const commonLib = require("./parts/common-lib.js");
         }
 
         for (let { module, name } of TRIGGER_BY_EVENT) {
-            if (name !== "crashes") {
+            if (name !== "crashes" && name !== "pii") {
                 try {
                     await module.triggerByEvent({ events, app });
                 }
@@ -31,6 +31,19 @@ const commonLib = require("./parts/common-lib.js");
         }
     });
 
+    plugins.register("/pii/incident", async function(ob) {
+        for (let { module, name } of TRIGGER_BY_EVENT) {
+            if (name === "pii") {
+                try {
+                    await module.triggerByEvent(ob.data);
+                }
+                catch (err) {
+                    log.e("Alert module 'pii' couldn't be triggered by event", err);
+                }
+            }
+        }
+    });
+
 }(exported));
 
 module.exports = exported;

plugins/alerts/api/jobs/AlertProcessor.js

@@ -17,6 +17,7 @@ const ALERT_MODULES = {
     "cohorts": require("../alertModules/cohorts.js"),
     "dataPoints": require("../alertModules/dataPoints.js"),
     "crashes": require("../alertModules/crashes.js"),
+    "pii": require("../alertModules/pii.js"),
 };
 
 /**

plugins/alerts/api/parts/common-lib.js

@@ -18,7 +18,7 @@
  * @property {boolean}       enabled           - true|false
  * @property {string}        compareDescribe   - text to show on lists for this alert
  * @property {Array<string>} alertValues       - audience e.g. for alertBy="email", list of e-mails
- * @property {Array<string>} allGroups         - 
+ * @property {Array<string>} allGroups         -
  * @property {string}        createdBy         - creation time
  */
 
@@ -81,6 +81,7 @@ const TRIGGERED_BY_EVENT = {
     nps: "new NPS response",
     rating: "new rating response",
     crashes: "new crash/error",
+    pii: "new sensitive data incident",
 };
 
 module.exports = {
@@ -207,7 +208,7 @@ async function compileEmail(result) {
  * Formats the metric value to ensure it maintains its type.
  * If the value is a number, it rounds to 2 decimal places if necessary.
  * Otherwise, it returns the value as is.
- * 
+ *
  * @param {number|string} value - The value to be formatted.
  * @returns {number|string} The formatted value, maintaining the original type.
  */

plugins/alerts/frontend/public/javascripts/countly.models.js

@@ -158,6 +158,38 @@
     }
     countlyAlerts.getRatingForApp = getRatingForApp;
 
+    /**
+     * Get PII rules for the specified app.
+     * @param {string} appId - The ID of the app.
+     * @param {function} callback - The callback function.
+     */
+    function getPiiRulesForApp(appId, callback) {
+        var data = {
+            enabled: "true",
+        };
+        if (appId === "all") {
+            data.app_ids = "all";
+            data.scope_filter = "global";
+        }
+        else {
+            data.app_ids = JSON.stringify([appId]);
+            data.include_global = "true";
+        }
+        $.ajax({
+            type: "GET",
+            url: countlyCommon.API_PARTS.data.r + "/pii/rules",
+            data: data,
+            dataType: "json",
+            success: function(res) {
+                if (res && Array.isArray(res)) {
+                    return callback(res);
+                }
+                return callback([]);
+            },
+        });
+    }
+    countlyAlerts.getPiiRulesForApp = getPiiRulesForApp;
+
     /**
      * extract getEventLongName
      * @param {string} eventKey - event key in db