Implemented a Streamflow Translator

pyproject.toml

@@ -26,6 +26,7 @@ license = {text = "GNU General Public License v3 (GPLv3)"}
 dependencies = [
     "jsonschema",
     "matplotlib",
+    "pygraphviz",
     "networkx",
     "numpy",
     "pandas",

tests/translators_loggers/Dockerfile.airflow

@@ -1,5 +1,5 @@
-# docker build -t wfcommons-dev -f Dockerfile.airflow .
-# docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash
+# docker build -t wfcommons-dev-airflow -f Dockerfile.airflow .
+# docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev-airflow /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -21,12 +21,14 @@ RUN apt-get -y install cmake-data
 RUN apt-get -y install sudo 
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
+RUN apt-get -y install graphviz libgraphviz-dev
+
 #RUN apt-get -y install gcc-multilib
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 

tests/translators_loggers/Dockerfile.bash

@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.cwl

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-cwl -f Dockerfile.cwl .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-cwl /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -22,12 +22,13 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.dask

@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.makeflow

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-makeflow -f Dockerfile.makeflow .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-makeflow /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -23,12 +23,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.nextflow

@@ -23,12 +23,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN pip install --break-system-packages pathos pandas filelock 
-RUN pip install --break-system-packages networkx scipy matplotlib
+RUN pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.parsl

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-parsl -f Dockerfile.parsl .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-parsl /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.pegasus

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-pegasus -f Dockerfile.pegasus .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-pegasus /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.pycompss

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-pycompss -f Dockerfile.pycompss .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-pycompss /bin/bash
 
 FROM compss/pycompss
 
@@ -22,12 +22,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install pathos pandas filelock
-RUN python3 -m pip install networkx scipy matplotlib
+RUN python3 -m pip install networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install pyyaml jsonschema requests
 #RUN python3 -m pip install --upgrade setuptools
 #RUN python3 -m pip install --upgrade wheels

tests/translators_loggers/Dockerfile.streamflow

@@ -0,0 +1,55 @@
+# docker build --platform amd64 -t wfcommons-dev-streamflow -f Dockerfile.streamflow .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-streamflow /bin/bash
+
+# FROM alphaunito/streamflow
+FROM amd64/ubuntu:noble
+
+LABEL org.containers.image.authors="henric@hawaii.edu"
+
+# update repositories
+RUN apt-get update
+
+# set timezone
+RUN echo "America/Los_Angeles" > /etc/timezone && export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata
+
+# install useful stuff
+RUN apt-get -y install pkg-config 
+RUN apt-get -y install git
+RUN apt-get -y install wget 
+RUN apt-get -y install make 
+RUN apt-get -y install cmake 
+RUN apt-get -y install cmake-data 
+RUN apt-get -y install sudo 
+RUN apt-get -y install vim --fix-missing
+RUN apt-get -y install gcc 
+RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
+
+
+# Python stuff
+RUN apt-get -y install python3 python3-pip
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+RUN python3 -m pip install --break-system-packages pathos pandas filelock
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
+RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
+
+# Stress-ng
+RUN apt-get -y install stress-ng
+
+# Streamflow
+RUN apt-get -y install nodejs
+RUN python3 -m pip install --break-system-packages streamflow==0.2.0.dev14
+
+# Add wfcommons user
+RUN useradd -ms /bin/bash wfcommons
+RUN adduser wfcommons sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+ENV PATH="$PATH:/home/wfcommons/.local/bin/"
+
+USER wfcommons
+WORKDIR /home/wfcommons
+# Making this directory world rwx to facilitate testing
+RUN chmod -R 777 /home/wfcommons
+

tests/translators_loggers/Dockerfile.swiftt

@@ -22,12 +22,13 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/Dockerfile.taskvine

@@ -1,5 +1,5 @@
-# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
-# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
+# docker build --platform amd64 -t wfcommons-dev-taskvine -f Dockerfile.taskvine .
+# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-taskvine /bin/bash
 
 FROM amd64/ubuntu:noble
 
@@ -23,12 +23,14 @@ RUN apt-get -y install sudo
 RUN apt-get -y install vim --fix-missing
 RUN apt-get -y install gcc 
 RUN apt-get -y install gcc-multilib
+RUN apt-get -y install graphviz libgraphviz-dev
+
 
 # Python stuff
 RUN apt-get -y install python3 python3-pip
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
-RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib pygraphviz
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
 

tests/translators_loggers/build_docker_docker_images.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-for backend in "dask" "parsl" "nextflow" "airflow" "bash" "taskvine" "makeflow" "cwl" "pegasus" "swiftt"; do
+for backend in "dask" "parsl" "nextflow" "airflow" "bash" "taskvine" "makeflow" "cwl" "pegasus" "swiftt" "streamflow"; do
   echo "Building $backend Docker image..."
   docker build --platform linux/amd64 -t wfcommons/wfcommons-testing-$backend -f Dockerfile.$backend .
 done

tests/translators_loggers/test_translators_loggers.py

@@ -34,12 +34,14 @@
 from wfcommons.wfbench import TaskVineTranslator
 from wfcommons.wfbench import MakeflowTranslator
 from wfcommons.wfbench import CWLTranslator
+from wfcommons.wfbench import StreamflowTranslator
 from wfcommons.wfbench import PegasusTranslator
 from wfcommons.wfbench import SwiftTTranslator
 
 from wfcommons.wfinstances import PegasusLogsParser
 from wfcommons.wfinstances.logs import TaskVineLogsParser
 from wfcommons.wfinstances.logs import MakeflowLogsParser
+from wfcommons.wfinstances.logs import ROCrateLogsParser
 
 
 def _create_workflow_benchmark() -> (WorkflowBenchmark, int):
@@ -114,6 +116,7 @@ def _additional_setup_swiftt(container):
     "taskvine": _additional_setup_taskvine,
     "makeflow": noop,
     "cwl": noop,
+    "streamflow": noop,
     "pegasus": _additional_setup_pegasus,
     "swiftt": _additional_setup_swiftt,
 }
@@ -189,6 +192,24 @@ def run_workflow_cwl(container, num_tasks, str_dirpath):
     # and there is a 2* because there is a message for the job and for the step)
     assert (output.decode().count("completed success") == 3 + 2 *num_tasks)
 
+def run_workflow_streamflow(container, num_tasks, str_dirpath):
+    # Run the workflow!
+    # Note that the input file is hardcoded and Blast-specific
+    exit_code, output = container.exec_run(cmd="streamflow run ./streamflow.yml",
+                                           user="wfcommons", stdout=True, stderr=True)
+    # print(output.decode())
+    # Check sanity
+    assert (exit_code == 0)
+    # 2 extra "COMPLETED Step" ("COMPLETED Step /compile_output_files", "COMPLETED Step /compile_log_files")
+    assert (output.decode().count("COMPLETED Step") == num_tasks + 2)
+
+    # Generate RO-Crate now that the workflow has completed (Fails for now)
+    exit_code, output = container.exec_run(cmd="streamflow list",
+                                           user="wfcommons", stdout=True, stderr=True)
+    uuid = output.decode().splitlines()[1].strip().split(" ")[0]
+    exit_code, output = container.exec_run(cmd=f"streamflow prov {uuid}",
+                                           user="wfcommons", stdout=True, stderr=True)
+
 def run_workflow_pegasus(container, num_tasks, str_dirpath):
     # Run the workflow!
     exit_code, output = container.exec_run(cmd="bash /home/wfcommons/run_workflow.sh",
@@ -217,6 +238,7 @@ def run_workflow_swiftt(container, num_tasks, str_dirpath):
     "taskvine": run_workflow_taskvine,
     "makeflow": run_workflow_makeflow,
     "cwl": run_workflow_cwl,
+    "streamflow": run_workflow_streamflow,
     "pegasus": run_workflow_pegasus,
     "swiftt": run_workflow_swiftt,
 }
@@ -231,6 +253,7 @@ def run_workflow_swiftt(container, num_tasks, str_dirpath):
     "taskvine": TaskVineTranslator,
     "makeflow": MakeflowTranslator,
     "cwl": CWLTranslator,
+    "streamflow": StreamflowTranslator,
     "pegasus": PegasusTranslator,
     "swiftt": SwiftTTranslator,
 }
@@ -251,6 +274,7 @@ class TestTranslators:
            "taskvine",
            "makeflow",
            "cwl",
+           "streamflow",
            "pegasus",
         ])
     @pytest.mark.unit
@@ -299,16 +323,17 @@ def test_translator(self, backend) -> None:
                                                stdout=True, stderr=True)
 
         # Run the log parser if any
+        parser = None
         if backend == "pegasus":
             parser = PegasusLogsParser(dirpath / "work/wfcommons/pegasus/Blast-Benchmark/run0001/")
         elif backend == "taskvine":
             parser = TaskVineLogsParser(dirpath / "vine-run-info/most-recent/vine-logs", filenames_to_ignore=["cpu-benchmark","stress-ng", "wfbench"])
         elif backend == "makeflow":
             parser = MakeflowLogsParser(execution_dir = dirpath, resource_monitor_logs_dir = dirpath / "monitor_data/")
-        else:
-            parser = None
+        # elif backend == "streamflow":
+        #     parser =ROCrateLogsParser(dirpath / "work/wfcommons/most-recent/wfbench")
 
-        if parser:
+        if parser is not None:
             sys.stderr.write(f"[{backend}] Parsing the logs...\n")
             reconstructed_workflow : Workflow = parser.build_workflow(f"reconstructed_workflow_{backend}")
             reconstructed_workflow.write_json(pathlib.Path("/tmp/reconstructed_workflow.json"))