o2dpg_workflow_runner.py: Adapt format for --update-resources to that of o2dpg_sim_metrics.py

Author: sawenzel

MC/bin/o2_dpg_workflow_runner.py

@@ -440,6 +440,7 @@ def getweight(tid):
 
 # update the resource estimates of a workflow based on resources given via JSON
 def update_resource_estimates(workflow, resource_json):
+    # the resource_dict here is generated by tool o2dpg_sim_metrics.py json-stat
     resource_dict = load_json(resource_json)
     stages = workflow["stages"]
 
@@ -455,17 +456,18 @@ def update_resource_estimates(workflow, resource_json):
         new_resources = resource_dict[name]
 
         # memory
-        newmem = new_resources.get("mem", None)
+        newmem = new_resources.get("pss", {}).get("max", None)
         if newmem is not None:
             oldmem = task["resources"]["mem"]
             actionlogger.info("Updating mem estimate for " + task["name"] + " from " + str(oldmem) + " to " + str(newmem))
             task["resources"]["mem"] = newmem
 
         # cpu
-        newcpu = new_resources.get("cpu", None)
+        newcpu = new_resources.get("cpu", {}).get("mean", None)
         if newcpu is not None:
             oldcpu = task["resources"]["cpu"]
             rel_cpu = task["resources"]["relative_cpu"]
+            # TODO: No longer sure about this since we inject numbers from actually measured workloads
             if rel_cpu is not None:
                # respect the relative CPU settings
                # By default, the CPU value in the workflow is already scaled if relative_cpu is given.
@@ -597,7 +599,7 @@ def __init__(self, tid, name, cpu, cpu_relative, mem, resource_boundaries):
         # sampled resources of this
         self.cpu_sampled = None
         self.mem_sampled = None
-        # Set these after a task has finished to compute new estomates for related tasks
+        # Set these after a task has finished to compute new estimates for related tasks
         self.walltime = None
         self.cpu_taken = None
         self.mem_taken = None

MC/utils/o2dpg_sim_metrics.py

@@ -1004,16 +1004,19 @@ def merge_stats_into(list_of_json_stats, outputfile, metadata):
   running = {}
   # read all the inputs
   for inp_json in list_of_json_stats:
+    # we may have to strip the meta-data section first of all
     running = merge_stats(inp_json, running)
 
   # attach meta-data
   running["meta-data"] = metadata
 
   # now write out the result into the output file
-  if running:
+  if running and outputfile != None:
      with open(outputfile, 'w') as f:
         json.dump(running, f)
 
+  return running
+
 
 def build_meta_header(stringarg):
   meta = {}