Merge branch 'main' into hattinf/O2B-719/add-range-based-filtering-for-run-number

Author: Fennne

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.4.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.4.0)
+* Notable changes for users:
+  * Fixed tag color not being updated when switching to run details after updating a tag
+  * Environment variable ENABLE_HOUSEKEEPING must be set to true (case-insensitive) to actually enable housekeeping
+  * Use time range picker for runs start & stop filtering
+  * Run details page has been modernized
+* Notable changes for developers:
+  * Runs and QC flags timestamps now store milliseconds
+  * Fixed users that start/stop runs not being extracted from kafka message
+  * Fixed TF timestamps being ignored when creating QC flags
+  * Fixed randomly failing test in FLP frontend tests
+  * Use coalesced run time in raw SQL querries
+  * Removed the max number of retries for kafka connection
+
 ## [1.3.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.3.0)
 * Notable changes for users:
   * Fixed physical constants values which resulted in wrong AVG center of mass energy

database/CHANGELOG.md

@@ -1,3 +1,24 @@
+## [1.4.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.4.0)
+* Changes made to the database
+  * Following columns in runs table has been changed to Datetime(3):
+    * time_o2_start
+    * time_trg_start
+    * first_tf_timestamp
+    * last_tf_timestamp
+    * time_trg_end
+    * time_o2_end
+  * Following columns in quality_control_flags table has been changed to Datetime(3):
+    * from
+    * to
+    * created_at
+    * updated_at
+  * Following columns in quality_control_flag_effective_periods table has been changed to Datetime(3):
+    * from
+    * to
+    * created_at
+    * updated_at
+  * Added two more virtual columns to runs, rct_time_start and rct_time_end that coalesce first/last TF timestamps with run start/stop
+
 ## [1.3.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.3.0)
 * Changes made to the database
   * Fixed physical constants values in database

docker-compose.yml

@@ -24,7 +24,8 @@ services:
       FLP_INFOLOGGER_URL: "${ALI_ECS_GUI_URL:-http://localhost:8081}"
       QC_GUI_URL: "${ALI_ECS_GUI_URL:-http://localhost:8082}"
       ALI_FLP_INDEX_URL: "${ALI_ECS_GUI_URL:-http://localhost:80}"
-      CCDB_ENABLE_SYNCHRONIZATION: false
+      CCDB_ENABLE_SYNCHRONIZATION: "${CCDB_ENABLE_SYNCHRONIZATION:-false}"
+      CCDB_RUN_INFO_URL: "${CCDB_RUN_INFO_URL:-}"
     links:
       - database
     restart: unless-stopped

lib/application.js

@@ -48,7 +48,6 @@ class BookkeepingApplication {
             const kafkaClient = new Kafka({
                 clientId: 'bookkeeping',
                 brokers: KafkaConfig.brokers,
-                retry: { retries: 3 },
                 logLevel: logLevel.NOTHING,
             });
 
@@ -94,15 +93,9 @@ class BookkeepingApplication {
                 const ccdbSynchronizer = new CcdbSynchronizer(ccdbConfig.runInfoUrl);
                 this.scheduledProcessesManager.schedule(
                     // Sync runs a few sync period ago in case some synchronization failed
-                    async () => {
-                        try {
-                            await ccdbSynchronizer.syncFirstAndLastTf(new Date(Date.now() - ccdbConfig.synchronizationPeriod * 5));
-                        } catch (e) {
-                            this._logger.errorMessage(`Failed to synchronize runs first and last TF:\n ${e.stack}`);
-                        }
-                    },
+                    () => ccdbSynchronizer.syncFirstAndLastTf(new Date(Date.now() - ccdbConfig.synchronizationPeriod * 5)),
                     {
-                        wait: 3 * 1000,
+                        wait: 10 * 1000,
                         every: ccdbConfig.synchronizationPeriod,
                     },
                 );

lib/config/services.js

@@ -26,7 +26,7 @@ const {
 } = process.env ?? {};
 
 exports.services = {
-    enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING ?? false,
+    enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING?.toLowerCase() === 'true',
     aliEcsGui: {
         url: process.env?.ALI_ECS_GUI_URL || null,
         token: process.env?.ALI_ECS_GUI_TOKEN || null,
@@ -64,7 +64,7 @@ exports.services = {
 
     ccdb: {
         enableSynchronization: CCDB_ENABLE_SYNCHRONIZATION?.toLowerCase() === 'true',
-        synchronizationPeriod: Number(CCDB_SYNCHRONIZATION_PERIOD) || 24 * 60 * 60 * 1000, // 1h in milliseconds
+        synchronizationPeriod: Number(CCDB_SYNCHRONIZATION_PERIOD) || 24 * 60 * 60 * 1000, // 1d in milliseconds
         runInfoUrl: CCDB_RUN_INFO_URL,
     },
 };

lib/database/migrations/20241212074001-create-rct-run-start-and-stop.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const ADD_RUN_RCT_START_AND_STOP = `
+    ALTER TABLE runs
+        ADD COLUMN qc_time_start DATETIME(3) AS (COALESCE(first_tf_timestamp, time_trg_start, time_o2_start)) VIRTUAL AFTER time_start,
+        ADD COLUMN qc_time_end   DATETIME(3) AS (COALESCE(last_tf_timestamp, time_trg_end, time_o2_end)) VIRTUAL AFTER qc_time_start;
+`;
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    up: async (queryInterface) => queryInterface.sequelize.query(ADD_RUN_RCT_START_AND_STOP),
+
+    down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => {
+        await queryInterface.removeColumn('runs', 'qc_time_start', { transaction });
+        await queryInterface.removeColumn('runs', 'qc_time_end', { transaction });
+    }),
+};

lib/database/repositories/QcFlagRepository.js

@@ -203,32 +203,17 @@ class QcFlagRepository extends Repository {
                 GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,
 
                 IF(
-                    (
-                        COALESCE(run.time_trg_end,   run.time_o2_end  ) IS NULL
-                     OR COALESCE(run.time_trg_start, run.time_o2_start) IS NULL
-                    ),
+                    run.time_start IS NULL OR run.time_end IS NULL,
                     IF(
-                        SUM(
-                            COALESCE(effectivePeriods.\`to\`  , 0)
-                          + COALESCE(effectivePeriods.\`from\`, 0)
-                        ) = 0,
+                        effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\`  IS NULL,
                         1,
                         null
                     ),
                     SUM(
-                        COALESCE(
-                            effectivePeriods.\`to\`,
-                            UNIX_TIMESTAMP(run.time_trg_end),
-                            UNIX_TIMESTAMP(run.time_o2_end)
-                        )
-                      - COALESCE(
-                            effectivePeriods.\`from\`,
-                            UNIX_TIMESTAMP(run.time_trg_start),
-                            UNIX_TIMESTAMP(run.time_o2_start)
-                        )
+                        COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end))
+                      - COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start))
                     ) / (
-                        UNIX_TIMESTAMP(COALESCE(run.time_trg_end,   run.time_o2_end))
-                      - UNIX_TIMESTAMP(COALESCE(run.time_trg_start, run.time_o2_start))
+                        UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
                     )
                 ) AS effectiveRunCoverage
 

…blic/views/Runs/Details/editTagsPanel.js …ents/Filters/RunsFilter/o2StartFilter.jslib/public/views/Runs/Details/editTagsPanel.js renamed to lib/public/components/Filters/RunsFilter/o2StartFilter.js lib/public/views/Runs/Details/editTagsPanel.js renamed to lib/public/components/Filters/RunsFilter/o2StartFilter.js

@@ -11,17 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-import { h } from '/js/src/index.js';
-import { tagPicker } from '../../../components/tag/tagPicker.js';
+import { timeRangeFilter } from '../common/filters/timeRangeFilter.js';
 
 /**
- * A panel containing:
- * a list of potential tags to add to a RUN
- * a button to update the tag selection
- * @param {RunDetailsModel} runDetailsModel the details model
- * @return {vnode} virtual node with representation of the panel
+ * Returns a filter to be applied on run start
+ *
+ * @param {RunsOverviewModel} runsOverviewModel the run overview model object
+ * @return {Component} the filter component
  */
-export const editTagsPanel = (runDetailsModel) => h(
-    '#tags-selection.flex-column.w-30.p2.g2',
-    tagPicker(runDetailsModel.editionTagPickerModel),
-);
+export const o2StartFilter = (runsOverviewModel) => timeRangeFilter(runsOverviewModel.o2StartFilterModel);

lib/public/components/Filters/RunsFilter/o2StopFilter.js

@@ -0,0 +1,21 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+import { timeRangeFilter } from '../common/filters/timeRangeFilter.js';
+
+/**
+ * Returns a filter to be applied on run stop
+ *
+ * @param {RunsOverviewModel} runsOverviewModel the run overview model object
+ * @return {Component} the filter component
+ */
+export const o2StopFilter = (runsOverviewModel) => timeRangeFilter(runsOverviewModel.o2stopFilterModel);