Merge pull request #383 from Baltic-RCC/dev

Release to production

Author: m-karo

.github/workflows/publish-workers.yml

@@ -19,6 +19,7 @@ jobs:
           - emfos-model-validator
           - emfos-model-merger
           - emfos-task-generator
+          - emfos-model-quality
 
     steps:
       - uses: actions/checkout@v4

config/cgm_worker/scaler.properties

@@ -1,4 +1,6 @@
 [MAIN]
 MAX_ITERATION = 15
 BALANCE_THRESHOLD = 2
+CONSTANT_POWER_FACTOR = False
+POWER_FACTOR_THRESHOLD = 1
 DEBUG = True

config/model_quality/model_quality.properties

@@ -1,5 +1,8 @@
 [MAIN]
-INPUT_RMQ_QUEUE = object-storage.models.quality
+INPUT_RMQ_QUEUE = object-storage.models.quality.cgm
 ELK_QUALITY_INDEX = emfos-model-quality
 ELK_STATISTICS_INDEX = emfos-model-statistics
-BORDER_LIMIT = 250
+BORDER_LIMIT = 250
+LINE_LIMIT_TEMPERATURE = 25 C
+IGM_RULE_SET = impedance,line_rating
+CGM_RULE_SET = kruonis,rtec,outage,lt_pl_xborder

config/model_validator/model_validator.properties

@@ -3,6 +3,9 @@ INPUT_RMQ_QUEUE = object-storage.models.validation
 OUTPUT_RMQ_EXCHANGE = emf-quality.test
 VALIDATION_ELK_INDEX = emfos-igm-validation
 METADATA_ELK_INDEX = emfos-opde-models
+ENABLE_LVL8_REPORTS = False
+QAS_EIC = EIC
+QAS_MSG_TYPE = MSG_TYPE
 ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source
 ELK_ID_HASHING = True
 ENABLE_DYNAMIC_VALIDATION_SETTINGS = False

emf/common/converters/iec_schedule_to_ndjson.py

@@ -99,7 +99,7 @@ def get_metadata_from_xml(xml, include_namespace=True, prefix_root=False):
     return properties_dict
 
 
-def parse_iec_xml(element_tree: bytes, return_values_per_mtu: bool = True, mtu_resolution: str = 'PT1H'):
+def parse_iec_xml(element_tree: bytes, return_values_per_mtu: bool = True, mtu_resolution: str = 'PT15M'):
     """Parses iec xml to dictionary, meta on the same row with value and start/end time"""
     # TODO make return_values_per_mtu argument in parameters
     # TODO - maybe first analyse the xml, by getting all elements and try to match names, ala point_element_name = unique_element_namelist.contains("point") etc.

emf/model_quality/model_statistics.py emf/common/helpers/statistics.pyemf/model_quality/model_statistics.py renamed to emf/common/helpers/statistics.py

@@ -1,6 +1,8 @@
 import logging
 import sys
 
+import pandas as pd
+
 logger = logging.getLogger(__name__)
 
 logging.basicConfig(
@@ -12,6 +14,7 @@
 
 def sum_on_KEY(data, KEY, precision=1):
     return round(data.query("KEY == @KEY").VALUE.astype(float).sum(), precision)
+
 def get_load_and_generation_ssh(data):
     logger.info("Getting Load and Generation data") # TODO add wrapper with timing and logging
     return {
@@ -133,7 +136,6 @@ def get_tieflow_data(data):
 
 
     # Add SV results
-    # if sv_results := data.type_tableview("SvPowerFlow") is not None:
     try:
         tieflow_data = tieflow_data.merge(data.type_tableview("SvPowerFlow"),
                                           left_on="TieFlow.Terminal",
@@ -151,9 +153,7 @@ def get_tieflow_data(data):
 
     # Fix some names
     tieflow_data = tieflow_data.rename(columns={
-        "IdentifiedObject.energyIdentCodeEic_Terminal": "IdentifiedObject.energyIdentCodeEic_ControlArea",
-        "IdentifiedObject.energyIdentCodeEic": "IdentifiedObject.energyIdentCodeEic_Line"
-    })
+        "IdentifiedObject.energyIdentCodeEic_Terminal": "IdentifiedObject.energyIdentCodeEic_ControlArea"})
 
     # Add cross borders data
     def merge_sort_strings(row, col1, col2, delimiter='-'):
@@ -173,10 +173,7 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None):
         # Use only Interchange Control Area Tieflows
         tieflow_type = "http://iec.ch/TC57/2013/CIM-schema-cim16#ControlAreaTypeKind.Interchange"
         tieflow_data = get_tieflow_data(data)
-        try:
-            tieflow_data = tieflow_data.query("`ControlArea.type` == @tieflow_type")
-        except:
-            tieflow_data = tieflow_data[tieflow_data['ControlArea.type'] == tieflow_type]
+        tieflow_data = tieflow_data[tieflow_data['ControlArea.type'] == tieflow_type]
 
 
     if load_and_generation is None or load_and_generation.empty:
@@ -189,22 +186,15 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None):
     tieflow_np = tieflow_data[data_columns].sum().to_dict()
 
     # Summing values where BoundaryPoint.isDirectCurrent is False
-    try:
-        tieflow_acnp = tieflow_data.query("`BoundaryPoint.isDirectCurrent` == False")[data_columns].sum().to_dict()
-    except:
-        tieflow_acnp = tieflow_data[tieflow_data['BoundaryPoint.isDirectCurrent'] == False][data_columns].sum().to_dict()
+    tieflow_acnp = tieflow_data[tieflow_data['BoundaryPoint.isDirectCurrent'] == False][data_columns].sum().to_dict()
 
     # Processing HVDC tieflow data
     try:
-        tieflow_hvdc = tieflow_data.query("`BoundaryPoint.isDirectCurrent` == True")[
-            ['IdentifiedObject.energyIdentCodeEic_Line'] + data_columns].set_index('IdentifiedObject.energyIdentCodeEic_Line').to_dict("index")
+        tieflow_hvdc = tieflow_data[tieflow_data['BoundaryPoint.isDirectCurrent'] == True][
+            ['IdentifiedObject.energyIdentCodeEic_Line'] + data_columns].set_index(
+            'IdentifiedObject.energyIdentCodeEic_Line').drop_duplicates().to_dict("index")
     except:
-        try:
-            tieflow_hvdc = tieflow_data[tieflow_data['BoundaryPoint.isDirectCurrent'] == True][
-                ['IdentifiedObject.energyIdentCodeEic_Line'] + data_columns].set_index(
-                'IdentifiedObject.energyIdentCodeEic_Line').to_dict("index")
-        except:
-            tieflow_hvdc = None
+        tieflow_hvdc = {}
 
     # Calculating total_load, generation, and net position
     load = load_and_generation["EnergyConsumer.p"].sum()
@@ -225,7 +215,6 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None):
         'tieflow_abs': tieflow_abs,
         'tieflow_np': tieflow_np,
         'tieflow_acnp': tieflow_acnp,
-        'tieflow_hvdc': tieflow_hvdc,
     }
 
     # Fixes for ELK data storage

emf/common/integrations/object_storage/schedules.py

@@ -31,9 +31,8 @@ def query_hvdc_schedules(time_horizon: str,
             area_eic_map = json.loads(f.read())
         hvdc_eic_map = {}
 
-    # Define utc start/end times from timestamp
-    utc_start = datetime.fromisoformat(scenario_timestamp) - timedelta(minutes=30)
-    utc_end = datetime.fromisoformat(scenario_timestamp) + timedelta(minutes=30)
+    # Define utc end time from timestamp
+    utc_end = datetime.fromisoformat(scenario_timestamp) + timedelta(minutes=15)
 
     # Define business type by time horizon
     business_type = "B63" if time_horizon in ["1D", "ID"] else "B67"
@@ -47,7 +46,7 @@ def query_hvdc_schedules(time_horizon: str,
     # Get HVDC schedules
     schedules_df = service.query_schedules_from_elk(
         index="emfos-schedules*",
-        utc_start=utc_start.isoformat(),
+        utc_start=scenario_timestamp,
         utc_end=utc_end.isoformat(),
         metadata=metadata,
         period_overlap=True,
@@ -101,9 +100,8 @@ def query_acnp_schedules(time_horizon: str,
         with open(config.paths.cgm_worker.default_area_eic_map, "rb") as f:
             area_eic_map = json.loads(f.read())
 
-    # Define utc start/end times from timestamp
-    utc_start = datetime.fromisoformat(scenario_timestamp) - timedelta(minutes=30)
-    utc_end = datetime.fromisoformat(scenario_timestamp) + timedelta(minutes=30)
+    # Define utc end time from timestamp
+    utc_end = datetime.fromisoformat(scenario_timestamp) + timedelta(minutes=15)
 
     # Define metadata dictionary
     metadata = {
@@ -114,7 +112,7 @@ def query_acnp_schedules(time_horizon: str,
     # Get AC area schedules
     schedules_df = service.query_schedules_from_elk(
         index="emfos-schedules*",
-        utc_start=utc_start.isoformat(),
+        utc_start=scenario_timestamp,
         utc_end=utc_end.isoformat(),
         metadata=metadata,
         period_overlap=True,

emf/common/loadflow_tool/loadflow_settings.py

@@ -15,26 +15,17 @@
 # TODO - NOT AVAILABLE - cim:PowerFlowSettings.transformerRatioTapControlPriority "1" ;
 # TODO - USE IN SCALING - eumd:PowerFlowSettings.maxIterationNumberAIC "15" ;
 
-#DEFAULT settings applicable for all processes
+# DEFAULT settings applicable for all processes
 OPENLOADFLOW_DEFAULT_PROVIDER = {
     'slackBusesIds': '',
     'lowImpedanceBranchMode': 'REPLACE_BY_ZERO_IMPEDANCE_LINE', 
     'voltageRemoteControl': 'True',
-
-    # Legacy, not found in code, can be removed
-    # 'throwsExceptionInCaseOfSlackDistributionFailure': 'False',
-
     'loadPowerFactorConstant': 'True',  # cim:PowerFlowSettings.loadVoltageDependency TODO - check this
     'plausibleActivePowerLimit': '5000.0',
     'slackBusPMaxMismatch': '0.1', #slackBusDistributionThreshold
     'voltagePerReactivePowerControl': 'False',
-
-    # Legacy, not found in code, can be removed, replaced with generator, transformer etc control
-    # 'reactivePowerRemoteControl': 'False',
-
     'newtonRaphsonConvEpsPerEq': '1.0E-4',
     'voltageTargetPriorities': 'GENERATOR,TRANSFORMER,SHUNT',
-    # 'voltageInitModeOverride': None,
     'transformerVoltageControlMode': 'AFTER_GENERATOR_VOLTAGE_CONTROL',  
     'shuntVoltageControlMode': 'INCREMENTAL_VOLTAGE_CONTROL',  
     'minPlausibleTargetVoltage': '0.8',
@@ -49,10 +40,6 @@
     # 'debugDir': '',
     'incrementalTransformerVoltageControlOuterLoopMaxTapShift': '3',  # TODO - check this
     'secondaryVoltageControl': 'False',
-
-    # Legacy, not found in code, can be removed
-    # 'controllerToPilotPointVoltageSensiEpsilon': '0.01',
-
     'reactiveLimitsMaxPqPvSwitch': '3',
     'newtonRaphsonStoppingCriteriaType': 'UNIFORM_CRITERIA',
     'maxActivePowerMismatch': '0.01',  # cim:PowerFlowSettings.activePowerTolerance
@@ -96,8 +83,7 @@
 
 
 # Deviation of default provider from the default
-#############
-#Used for CGM main merging  process
+## Used for CGM main merging  process
 __IGM_VALIDATION_PROVIDER = {
     'slackBusSelectionMode': 'MOST_MESHED',
     'referenceBusSelectionMode':'GENERATOR_REFERENCE_PRIORITY',
@@ -130,12 +116,11 @@
     'maxAngleMismatch': '1.0E-5',  # cim:PowerFlowSettings.voltageAngleLimit "10" TODO - How to convert
     'slackBusPMaxMismatch': '0.09',  # To fulfill QOCDC SV_INJECTION_LIMIT = 0.1'
     'disableVoltageControlOfGeneratorsOutsideActivePowerLimits': 'true', # supress q part of igm-ssh-vs-cgm-ssh error
-
-     # 'extrapolateReactiveLimits': 'true',
     'disableInconsistentVoltageControls': 'true',
     'transformerVoltageControlMode': 'INCREMENTAL_VOLTAGE_CONTROL',
     'shuntVoltageControlMode': 'INCREMENTAL_VOLTAGE_CONTROL',
     'phaseShifterControlMode': 'INCREMENTAL',
+
 }
 __EU_RELAXED_PROVIDER = {
     'slackBusSelectionMode': 'MOST_MESHED',
@@ -153,15 +138,13 @@
     'maxAngleMismatch': '1.0E-5',  # cim:PowerFlowSettings.voltageAngleLimit "10" ; TODO - How to convert
     'slackBusPMaxMismatch': '0.09',  # To fulfill QOCDC SV_INJECTION_LIMIT = 0.1
     'disableVoltageControlOfGeneratorsOutsideActivePowerLimits': 'true', # supress q part of igm-ssh-vs-cgm-ssh error
-
     'disableInconsistentVoltageControls': 'true',
     'transformerVoltageControlMode': 'INCREMENTAL_VOLTAGE_CONTROL',
     'shuntVoltageControlMode': 'INCREMENTAL_VOLTAGE_CONTROL',
     'phaseShifterControlMode': 'INCREMENTAL',
 }
 
-
-#Baltic merge parameters
+## Baltic merge parameters
 __BA_DEFAULT_PROVIDER = {
     'slackBusSelectionMode': 'MOST_MESHED',
     'generatorReactivePowerRemoteControl': 'True',
@@ -206,31 +189,26 @@
     'slackBusPMaxMismatch': '0.09',  # To fulfill QOCDC SV_INJECTION_LIMIT = 0.1
     'disableVoltageControlOfGeneratorsOutsideActivePowerLimits': 'true', # supress q part of igm-ssh-vs-cgm-ssh error
 }
-############
 
 # Preparing PROVIDER settings options from default settings
-#############
-#Used for CGM main merging  process
+## Used for CGM main merging process
 IGM_VALIDATION_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 IGM_VALIDATION_PROVIDER.update(__IGM_VALIDATION_PROVIDER)
 EU_DEFAULT_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 EU_DEFAULT_PROVIDER.update(__EU_DEFAULT_PROVIDER)
 EU_RELAXED_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 EU_RELAXED_PROVIDER.update(__EU_RELAXED_PROVIDER)
 
-#Baltic merge parameters
+## Baltic merge parameters
 BA_DEFAULT_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 BA_DEFAULT_PROVIDER.update(__BA_DEFAULT_PROVIDER)
 BA_RELAXED_1_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 BA_RELAXED_1_PROVIDER.update(__BA_RELAXED_1_PROVIDER)
 BA_RELAXED_2_PROVIDER = OPENLOADFLOW_DEFAULT_PROVIDER.copy()
 BA_RELAXED_2_PROVIDER.update(__BA_RELAXED_2_PROVIDER)
-##############
-
 
 # Prepare pypowsybl loadflow parameters classes
-##############
-#Used for CGM main merging  process
+## Used for CGM main merging  process
 IGM_VALIDATION = pypowsybl.loadflow.Parameters(
     #voltage_init_mode=pypowsybl._pypowsybl.VoltageInitMode.UNIFORM_VALUES,  # cim:PowerFlowSettings.flatStart "true"
     transformer_voltage_control_on=True,  # cim:PowerFlowSettings.transformerRatioTapControlPriority "1"
@@ -247,6 +225,7 @@
     connected_component_mode=pypowsybl._pypowsybl.ConnectedComponentMode.MAIN,
     provider_parameters=IGM_VALIDATION_PROVIDER,
 )
+
 EU_DEFAULT = pypowsybl.loadflow.Parameters(
     #voltage_init_mode=pypowsybl._pypowsybl.VoltageInitMode.UNIFORM_VALUES,  # cim:PowerFlowSettings.flatStart "true"
     transformer_voltage_control_on=True,  # @cim:PowerFlowSettings.transformerRatioTapControlPriority": "1"
@@ -263,6 +242,7 @@
     connected_component_mode=pypowsybl._pypowsybl.ConnectedComponentMode.ALL,
     provider_parameters=EU_DEFAULT_PROVIDER,
 )
+
 EU_RELAXED = pypowsybl.loadflow.Parameters(
     #voltage_init_mode=pypowsybl._pypowsybl.VoltageInitMode.UNIFORM_VALUES,  # cim:PowerFlowSettings.flatStart "true"
     transformer_voltage_control_on=True,  # cim:PowerFlowSettings.transformerRatioTapControlPriority "0"
@@ -280,7 +260,7 @@
     provider_parameters=EU_RELAXED_PROVIDER,
 )
 
-#Baltic merge parameters
+## Baltic merge parameters
 BA_DEFAULT = pypowsybl.loadflow.Parameters(
     voltage_init_mode=pypowsybl._pypowsybl.VoltageInitMode.UNIFORM_VALUES,  # cim:PowerFlowSettings.flatStart "true"
     transformer_voltage_control_on=True,  # @cim:PowerFlowSettings.transformerRatioTapControlPriority": "1"
@@ -297,6 +277,7 @@
     connected_component_mode=pypowsybl._pypowsybl.ConnectedComponentMode.ALL,
     provider_parameters=BA_DEFAULT_PROVIDER,
 )
+
 BA_RELAXED_1 = pypowsybl.loadflow.Parameters(
     voltage_init_mode=pypowsybl._pypowsybl.VoltageInitMode.UNIFORM_VALUES,  # cim:PowerFlowSettings.flatStart "true"
     transformer_voltage_control_on=False,  # cim:PowerFlowSettings.transformerRatioTapControlPriority "0"
@@ -330,4 +311,3 @@
     connected_component_mode=pypowsybl._pypowsybl.ConnectedComponentMode.ALL,
     provider_parameters=BA_RELAXED_2_PROVIDER,
 )
-#################

emf/model_merger/model_merger.py

@@ -404,19 +404,22 @@ def handle(self, task_object: dict, properties: dict, **kwargs):
         logger.debug(f"Post processing took: {(post_p_end - post_p_start).total_seconds()} seconds")
         logger.debug(f"Merging took: {(merge_end - merge_start).total_seconds()} seconds")
 
-        # Upload to OPDM
+        # Upload to OPDM 
         if model_upload_to_opdm:
-            try:
-                self.opdm_service = opdm.OPDM()
-                for item in serialized_data:
-                    logger.info(f"Uploading to OPDM: {item.name}")
-                    time.sleep(2)
-                    async_call(function=self.opdm_service.publication_request,
-                               callback=log_opdm_response,
-                               file_path_or_file_object=item)
-                merged_model.uploaded_to_opde = True
-            except Exception as error:
-                logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True)
+            if merged_model.loadflow[0]['status'] == 'CONVERGED':  # Only upload if the model LF is solved
+                try:
+                    self.opdm_service = opdm.OPDM()
+                    for item in serialized_data:
+                        logger.info(f"Uploading to OPDM: {item.name}")
+                        time.sleep(4)
+                        async_call(function=self.opdm_service.publication_request,
+                                callback=log_opdm_response,
+                                file_path_or_file_object=item)
+                    merged_model.uploaded_to_opde = True
+                except Exception as error:
+                    logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True)
+            else:
+                logger.info(f"Model not uploaded to OPDM due to convergance issue: {merged_model.loadflow[0]['status']}")
 
         # Create zipped model data
         merged_model_object = BytesIO()