Implement sensitivity analysis

fastapi_app/Dockerfile

@@ -13,6 +13,7 @@ RUN pip install -r /tmp/requirements.txt
 # install multi-vector-simulator's version pinned in docker-compose.yml file
 RUN pip install multi-vector-simulator==$mvs_version
 RUN pip install gunicorn
+RUN pip install jinja2==3.0
 
 COPY . /fastapi_app
 

fastapi_app/requirements.txt

@@ -1,6 +1,5 @@
 fastapi
 aiofiles
-jinja2==3.0
 uvicorn
 celery
 redis

fastapi_app/templates/index.html

@@ -39,6 +39,15 @@ <h3>Actual multi-vector-simulator's version for open_plan: {{ mvs_open_plan_vers
         <input name="json_file" type="file" multiple>
         <input value="Run simulation" type="submit">
     </form>
+
+    <p>Upload a mvs input json file, formated as epa and click on "Run sensitivity analysis"</p>
+    <form action="/uploadjson-sensitivity-analysis/open_plan" enctype="multipart/form-data" method="post">
+        <!-- the value of `name` property should be used in the uploadjson endpoint function -->
+        <input name="json_file" type="file" multiple>
+        <input value="Run sensitivity analysis" type="submit">
+    </form>
+
+
 </div>
 
 

fastapi_app/templates/submitted_sensitivity_analysis.html

@@ -0,0 +1,16 @@
+{% extends 'index.html' %}
+
+{% block body %}
+
+<div>
+    <h2>Sensitivity Analysis</h2>
+    {% if task_id %}
+        <p>
+            {{ task_id }}: <a href="{{ url_for('check_sensitivity_analysis', task_id=task_id) }}">results</a>
+        </p>
+    {% else %}
+        <p> The input parameters were not found in the input json file</p>
+    {% endif %}
+</div>
+
+{% endblock body %}

fastapi_app/webapp.py

@@ -1,3 +1,4 @@
+import copy
 import os
 import json
 import io
@@ -57,10 +58,10 @@
 
 
 @app.get("/")
-def index(request: Request) -> Response:
+def index(request: Request, template: str = "index.html") -> Response:
 
     return templates.TemplateResponse(
-        "index.html",
+        template,
         {
             "request": request,
             "mvs_dev_version": MVS_DEV_VERSION,
@@ -92,6 +93,18 @@ async def simulate_json_variable_open_plan(request: Request):
     return await simulate_json_variable(request, queue="open_plan")
 
 
+@app.post("/sendjson/openplan/sensitivity-analysis")
+async def sensitivity_analysis_json_variable_open_plan(request: Request):
+
+    input_dict = await request.json()
+
+    sensitivity_analysis_id = run_sensitivity_analysis(
+        input_json=json.dumps(input_dict)
+    )
+
+    return await check_sensitivity_analysis(sensitivity_analysis_id)
+
+
 @app.post("/uploadjson/dev")
 def simulate_uploaded_json_files_dev(
     request: Request, json_file: UploadFile = File(...)
@@ -116,6 +129,38 @@ def simulate_uploaded_json_files_open_plan(
     return run_simulation_open_plan(request, input_json=json_content)
 
 
+@app.post("/uploadjson-sensitivity-analysis/open_plan")
+def sensitivity_analysis_uploaded_json_files_open_plan(
+    request: Request, json_file: UploadFile = File(...)
+) -> Response:
+    """Receive mvs sensitivity analysis parameter in json post request and send it to simulator
+    the value of `name` property of the input html tag should be `json_file` as the second
+    argument of this function
+    """
+    json_content = jsonable_encoder(json_file.file.read())
+
+    sensitivity_analysis_id = run_sensitivity_analysis(input_json=json_content)
+
+    return templates.TemplateResponse(
+        "submitted_sensitivity_analysis.html",
+        {"request": request, "task_id": sensitivity_analysis_id},
+    )
+
+
+def run_sensitivity_analysis(input_json=None, queue="open_plan"):
+    """Send a sensitivity analysis task to a celery worker"""
+
+    """Receive mvs simulation parameter in json post request and send it to simulator"""
+    input_dict = json.loads(input_json)
+
+    sensitivity_analysis = celery_app.send_task(
+        f"{queue}.run_sensitivity_analysis", args=[input_dict], queue=queue, kwargs={}
+    )
+    answer = sensitivity_analysis.id
+
+    return answer
+
+
 def run_simulation(request: Request, input_json=None, queue="dev") -> Response:
     """Send a simulation task to a celery worker"""
 
@@ -157,9 +202,7 @@ async def check_task(task_id: str) -> JSONResponse:
         "status": res.state,
         "results": None,
     }
-    if res.state == states.PENDING:
-        task["status"] = res.state
-    else:
+    if res.state != states.PENDING:
         task["status"] = "DONE"
         results_as_dict = json.loads(res.result)
         server_info = results_as_dict.pop("SERVER")
@@ -173,6 +216,65 @@ async def check_task(task_id: str) -> JSONResponse:
     return JSONResponse(content=jsonable_encoder(task))
 
 
+@app.get("/check-sensitivity-analysis/{task_id}")
+async def check_sensitivity_analysis(task_id: str) -> JSONResponse:
+    sensitivity_analysis = celery_app.AsyncResult(task_id)
+
+    task = {
+        "server_info": None,
+        "mvs_version": None,
+        "id": task_id,
+        "status": sensitivity_analysis.state,
+        "results": dict(reference_simulation_id=None, sensitivity_analysis_steps=None),
+    }
+    if sensitivity_analysis.state != states.PENDING:
+        sa_results = sensitivity_analysis.result
+        if "ERROR" in sa_results:
+            task["status"] = "ERROR"
+            task["results"] = sa_results
+        else:
+            # fetch results of each sensitivity analysis steps
+            sa_step_ids = sa_results["sensitivity_analysis_ids"]
+
+            if "ERROR" in sa_step_ids:
+                task["status"] = "ERROR"
+                task["results"]["sensitivity_analysis_ids"] = sa_step_ids
+                answers = None
+            else:
+                server_info = None
+                answers = []
+                for sa_step_id in sa_step_ids:
+                    sa_step = celery_app.AsyncResult(sa_step_id)
+                    if sa_step.ready():
+                        results_as_dict = json.loads(sa_step.result)
+                        server_info = results_as_dict.pop("SERVER")
+                        if "ERROR" in results_as_dict:
+
+                            temp = copy.deepcopy(results_as_dict)
+                            temp.pop("step_idx")
+                            results_as_dict["output_values"] = temp
+                        answers.append(results_as_dict)
+                    else:
+                        task["status"] = states.PENDING
+                        answers = None
+                        break
+
+            if answers is not None:
+
+                task["status"] = "DONE"
+                task["server_info"] = server_info
+                task["mvs_version"] = MVS_SERVER_VERSIONS.get(server_info, "unknown")
+                task["results"]["sensitivity_analysis_steps"] = [
+                    d["output_values"]
+                    for d in sorted(answers, key=lambda item: item["step_idx"])
+                ]
+            # the "result" of the main simulation here is the mvs token leading to the simulations results
+            # that can be checked with url /check/{task_id}
+            task["results"]["reference_simulation_id"] = sa_results["ref_sim_id"]
+
+    return JSONResponse(content=jsonable_encoder(task))
+
+
 @app.get("/get_lp_file/{task_id}")
 async def get_lp_file(task_id: str) -> Response:
     res = celery_app.AsyncResult(task_id)
@@ -212,4 +314,3 @@ async def get_lp_file(task_id: str) -> Response:
             response = "There is no LP file output, did you check the LP file option when you started your simulation?"
 
     return response
-

task_queue/requirements.txt

@@ -2,3 +2,4 @@ celery
 flower
 redis
 python-multipart
+jsonschema

task_queue/sensitivity_analysis_utils.py

@@ -0,0 +1,145 @@
+import jsonschema
+import traceback
+from multi_vector_simulator.utils.data_parser import MAP_EPA_MVS
+
+
+SA_SCHEMA = {
+    "type": "object",
+    "required": [
+        "variable_parameter_name",
+        "variable_parameter_range",
+        "variable_parameter_ref_val",
+        "output_parameter_names",
+    ],
+    "properties": {
+        "variable_parameter_name": {
+            "oneOf": [{"type": "array", "items": {"type": "string"}}]
+        },
+        "variable_parameter_range": {"type": "array", "items": {"type": "number"}},
+        "variable_parameter_ref_val": {"type": "number"},
+        "output_parameter_names": {"type": "array", "items": {"type": "string"}},
+    },
+    "additionalProperties": False,
+}
+
+SENSITIVITY_ANALYSIS_SETTINGS = "sensitivity_analysis_settings"
+
+
+class SensitivityAnalysis:
+    __raw_input = None
+    variable_parameter_name = None
+    variable_parameter_range = None
+    variable_parameter_ref_val = None
+    output_parameter_names = None
+    validation_error = ""
+
+    def __init__(self, dict_settings):
+        sa_settings = dict_settings.get(SENSITIVITY_ANALYSIS_SETTINGS, None)
+        self.__raw_input = sa_settings
+        if sa_settings is not None:
+            try:
+                jsonschema.validate(sa_settings, SA_SCHEMA)
+                schema_is_valid = True
+            except jsonschema.exceptions.ValidationError as e:
+                schema_is_valid = False
+                self.validation_error = "{}".format(traceback.format_exc())
+
+            if schema_is_valid is True:
+                self.variable_parameter_name = sa_settings.get(
+                    "variable_parameter_name", None
+                )
+                self.format_parameter_name()
+
+                self.variable_parameter_range = sa_settings.get(
+                    "variable_parameter_range", None
+                )
+                self.output_parameter_names = sa_settings.get(
+                    "output_parameter_names", None
+                )
+                self.variable_parameter_ref_val = sa_settings.get(
+                    "variable_parameter_ref_val", None
+                )
+        else:
+            self.validation_error = (
+                f"The key {SENSITIVITY_ANALYSIS_SETTINGS} is missing in the input json"
+            )
+
+    def format_parameter_name(self):
+        if self.variable_parameter_name is not None:
+            self.variable_parameter_name = tuple(
+                [MAP_EPA_MVS.get(key, key) for key in self.variable_parameter_name]
+            )
+
+    def is_valid(self):
+        if (
+            self.variable_parameter_name is not None
+            and self.variable_parameter_range is not None
+            and self.output_parameter_names is not None
+            and self.variable_parameter_ref_val
+        ):
+            answer = True
+            if self.variable_parameter_ref_val not in self.variable_parameter_range:
+                answer = False
+                self.validation_error = (
+                    f"The value ({self.variable_parameter_ref_val}) of the variable parameter"
+                    f" {'.'.join(self.variable_parameter_name)} for the reference scenario of"
+                    " the sensitivity analysis is not within the variable range provided"
+                    f": [{', '.join([str(p) for p in self.variable_parameter_range]) }]"
+                )
+        else:
+            answer = False
+        return answer
+
+    def __str__(self):
+        return str(self.__raw_input)
+
+
+if __name__ == "__main__":
+    import json
+
+    with open("test_sa.json", "r") as jf:
+        input_dict = json.load(jf)
+
+    # with open("AFG_epa_format.json", "w") as jf:
+    #     json.dump(input_dict, jf, indent=4)
+
+    # input_dict[SENSITIVITY_ANALYSIS_SETTINGS] = {
+    #     "variable_parameter_name": ["energy_busses"],
+    #     "variable_parameter_range": [1, 2, 3.2, 3.5],
+    #     "variable_parameter_ref_val": 3,
+    #     "output_parameter_names": [
+    #         "specific_emissions_per_electricity_equivalent",
+    #         "total_feedinElectricity",
+    #         "total_internal_generation",
+    #         "peak_flow",
+    #     ],
+    # }
+    sa = SensitivityAnalysis(input_dict)
+    print(sa.is_valid(), sa.validation_error)
+    import ipdb
+
+    ipdb.set_trace()
+    # input_dict[SENSITIVITY_ANALYSIS_SETTINGS] = {
+    #     "variable_parameter_name": [
+    #         "energy_providers",
+    #         "Grid_DSO",
+    #         "energy_price",
+    #         "value",
+    #     ],
+    #     "variable_parameter_range": [1, 2, 3, 3.5],
+    #     "variable_parameter_ref_val": 3,
+    #     "output_parameter_names": [
+    #         "specific_emissions_per_electricity_equivalent",
+    #         "total_feedinElectricity",
+    #         "total_internal_generation",
+    #         "peak_flow",
+    #     ],
+    # # }
+    # sa = SensitivityAnalysis(input_dict)
+    # print(sa.is_valid(), sa.validation_error)
+    # sa = SensitivityAnalysis({SENSITIVITY_ANALYSIS_SETTINGS: {
+    #     "variable_parameter_name": "",
+    #     "variable_parameter_range": "",
+    #     "variable_parameter_ref_val": "",
+    #     "output_parameter_names": "",
+    # }})