updating branch to match main

httomo/pipelines/visr.json

@@ -0,0 +1,79 @@
+[
+    {
+        "method": "standard_tomo",
+        "module_path": "httomo.data.hdf.loaders",
+        "parameters": {
+            "data_path": "auto",
+            "image_key_path": "auto",
+            "rotation_angles": "auto"
+        }
+    },
+    {
+        "method": "remove_outlier",
+        "module_path": "httomolibgpu.misc.corr",
+        "parameters": {
+            "kernel_size": 3,
+            "dif": 1000
+        }
+    },
+    {
+        "method": "dark_flat_field_correction",
+        "module_path": "httomolibgpu.prep.normalize",
+        "parameters": {
+            "flats_multiplier": 1,
+            "darks_multiplier": 1,
+            "cutoff": 10
+        }
+    },
+    {
+        "method": "find_center_vo",
+        "module_path": "httomolibgpu.recon.rotation",
+        "parameters": {
+            "ind": null,
+            "average_radius": 0,
+            "cor_initialisation_value": null,
+            "smin": -50,
+            "smax": 50,
+            "srad": 6,
+            "step": 0.5,
+            "ratio": 0.5,
+            "drop": 20
+        },
+        "id": "centering",
+        "side_outputs": {
+            "cor": "centre_of_rotation"
+        }
+    },
+    {
+        "method": "FBP3d_tomobar",
+        "module_path": "httomolibgpu.recon.algorithm",
+        "parameters": {
+            "center": "${{centering.side_outputs.centre_of_rotation}}",
+            "detector_pad": false,
+            "filter_freq_cutoff": 0.35,
+            "recon_size": null,
+            "recon_mask_radius": 0.95
+        },
+        "save_result": false
+    },
+    {
+        "method": "calculate_stats",
+        "module_path": "httomo.methods",
+        "parameters": {},
+        "id": "statistics",
+        "side_outputs": {
+            "glob_stats": "glob_stats"
+        }
+    },
+    {
+        "method": "rescale_to_int",
+        "module_path": "httomolib.misc.rescale",
+        "parameters": {
+            "perc_range_min": 0,
+            "perc_range_max": 100,
+            "bits": 8,
+            "glob_stats": "${{statistics.side_outputs.glob_stats}}"
+        },
+        "save_result": true
+    }
+]

httomo/templates/visr-tomo-recon.yaml

@@ -0,0 +1,183 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ClusterWorkflowTemplate
+metadata:
+  name: visr-tomo-recon
+  labels:
+    workflows.diamond.ac.uk/science-group-imaging: "true"
+  annotations:
+    workflows.argoproj.io/title: ViSR Tomography Reconstruction
+    workflows.argoproj.io/description: |
+      Run a simple tomography reconstruction pipeline for data collected on
+      ViSR, and convert the output recon hdf5 data to raw binary data.
+    workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
+spec:
+  entrypoint: run
+  volumeClaimTemplates:
+  - metadata:
+      name: tmpdir
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 1Gi
+      storageClassName: netapp
+  arguments:
+    parameters:
+    - name: visitdir
+      valueFrom:
+        configMapKeyRef:
+          name: sessionspaces
+          key: data_directory
+  templates:
+  - name: run
+    inputs:
+      parameters:
+      - name: input
+      - name: output
+      - name: recon-outdir-name
+        value: recon-dir
+    steps:
+    - - name: tomo-recon
+        template: httomo
+        arguments:
+          parameters:
+          - name: config
+            value: |
+              {{- .Files.Get "pipelines/visr.json" | nindent 14 }}
+          - name: input
+            value: "{{`{{ inputs.parameters.input }}`}}"
+          - name: output
+            value: "{{`{{ inputs.parameters.output }}`}}"
+          - name: recon-outdir-name
+            value: "{{`{{ inputs.parameters.recon-outdir-name }}`}}"
+    - - name: generate-recon-artifact
+        template: convert-recon-data-format
+        arguments:
+          parameters:
+          - name: recon-dir-path
+            value: "{{`{{ inputs.parameters.output }}`}}/{{`{{ inputs.parameters.recon-outdir-name }}`}}"
+  - name: httomo
+    inputs:
+      parameters:
+      - name: config
+      - name: input
+      - name: output
+      - name: recon-outdir-name
+      - name: nprocs
+        value: 1
+      - name: memory
+        value: 1Gi
+    container:
+      image: ghcr.io/diamondlightsource/httomo:latest
+      env:
+      - name: CUPY_CACHE_DIR
+        value: /tmp/.cupy/kernel_cache
+      - name: MKL_NUM_THREADS
+        value: "1"
+      - name: NUMEXPR_NUM_THREADS
+        value: "1"
+      - name: OMP_NUM_THREADS
+        value: "1"
+      command:
+      - /opt/conda/bin/mpirun
+      args:
+      - "-n"
+      - "{{`{{ inputs.parameters.nprocs }}`}}"
+      - /opt/conda/bin/python
+      - "-m"
+      - "httomo"
+      - "run"
+      - "--pipeline-format"
+      - "json"
+      - "--output-folder-name"
+      - "{{`{{ inputs.parameters.recon-outdir-name }}`}}"
+      - "{{`{{ inputs.parameters.input }}`}}"
+      - "{{`{{ inputs.parameters.config }}`}}"
+      - "{{`{{ inputs.parameters.output }}`}}"
+      volumeMounts:
+      - name: session
+        mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
+      - name: tmpdir
+        mountPath: /tmp
+    podSpecPatch: |
+      containers:
+      - name: main
+        resources:
+          requests:
+            cpu: "{{`{{ inputs.parameters.nprocs }}`}}"
+            memory: "{{`{{ inputs.parameters.memory }}`}}"
+            nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}"
+          limits:
+            cpu: "{{`{{ inputs.parameters.nprocs }}`}}"
+            memory: "{{`{{ inputs.parameters.memory }}`}}"
+            nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}"
+    tolerations:
+    - key: nvidia.com/gpu
+      operator: Exists
+      effect: NoSchedule
+    - key: nodetype
+      operator: Equal
+      value: gpu
+      effect: NoSchedule
+    - key: nodegroup
+      operator: Equal
+      value: workflows
+      effect: NoSchedule
+    volumes:
+    - name: session
+      hostPath:
+        path:  "{{`{{ workflow.parameters.visitdir }}`}}"
+        type: Directory
+  - name: convert-recon-data-format
+    inputs:
+      parameters:
+      - name: recon-dir-path
+      - name: tmpdir-path
+        value: /tmp
+      - name: raw-recon-filename
+        value: recon.raw
+      - name: metadata-filename
+        value: metadata.json
+    script:
+      image: ghcr.io/diamondlightsource/httomo:latest
+      command: [/opt/conda/bin/python]
+      source: |
+        from pathlib import Path
+        import h5py
+        import json
+
+        RAW_RECON_PATH = "{{`{{ inputs.parameters.tmpdir-path }}`}}/{{`{{ inputs.parameters.raw-recon-filename }}`}}"
+        HDF5_RECON_DIR = Path("{{`{{ inputs.parameters.recon-dir-path }}`}}")
+        HDF5_RECON_FILENAME_PATTERN = "*-httomolib-rescale_to_int.h5"
+        hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0]
+
+        with h5py.File(hdf5_recon_data_path, "r") as f:
+            data = f["/data"][:]
+            data.tofile(RAW_RECON_PATH)
+
+        METADATA_PATH = "{{`{{ inputs.parameters.tmpdir-path }}`}}/{{`{{ inputs.parameters.metadata-filename }}`}}"
+
+        order = "C" if data.flags.c_contiguous else "F"
+        metadata = {"shape": list(data.shape), "dtype": str(data.dtype), "order": order}
+        with open(METADATA_PATH, "w") as f:
+            f.write(json.dumps(metadata, indent=2))
+      volumeMounts:
+      - name: session
+        mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
+      - name: tmpdir
+        mountPath: "{{`{{ inputs.parameters.tmpdir-path }}`}}"
+    volumes:
+    - name: session
+      hostPath:
+        path:  "{{`{{ workflow.parameters.visitdir }}`}}"
+        type: Directory
+    outputs:
+      artifacts:
+      - name: recon
+        path: "{{`{{ inputs.parameters.tmpdir-path }}`}}/{{`{{ inputs.parameters.raw-recon-filename }}`}}"
+        archive:
+          none: { }
+      - name: metadata
+        path: "{{`{{ inputs.parameters.tmpdir-path }}`}}/{{`{{ inputs.parameters.metadata-filename }}`}}"
+        archive:
+          none: { }