Merge branch 'main' into repost-control-db and fix conflict

stephen-riggs · stephen-riggs · commit 8abb29c4e819 · 2025-09-09T17:19:48.000+01:00
diff --git a/.bumpclient.toml b/.bumpclient.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.20.2"
+current_version = "0.20.3"
 commit = true
 tag = false
 
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.20.2"
+current_version = "0.20.3"
 commit = true
 tag = true
 
diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: murfey-services
 description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
-version: 0.20.2
+version: 0.20.3
 dependencies:
   - name: murfey-instrument-server-clem
   - name: murfey-instrument-server-tem
diff --git a/Helm/charts/murfey-instrument-server-clem/Chart.yaml b/Helm/charts/murfey-instrument-server-clem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-clem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.20.2
+version: 0.20.3
diff --git a/Helm/charts/murfey-instrument-server-tem/Chart.yaml b/Helm/charts/murfey-instrument-server-tem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-tem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.20.2
+version: 0.20.3
diff --git a/Helm/charts/murfey-rsync/Chart.yaml b/Helm/charts/murfey-rsync/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-rsync
 description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system
-version: 0.20.2
+version: 0.20.3
diff --git a/Helm/charts/murfey-server/Chart.yaml b/Helm/charts/murfey-server/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-server
 description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system
-version: 0.20.2
+version: 0.20.3
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.20.2"
+version = "0.20.3"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
diff --git a/src/murfey/__init__.py b/src/murfey/__init__.py
@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.20.2"
-__supported_client_version__ = "0.20.2"
+__version__ = "0.20.3"
+__supported_client_version__ = "0.20.3"
diff --git a/src/murfey/client/destinations.py b/src/murfey/client/destinations.py
@@ -47,11 +47,12 @@ def determine_default_destination(
                     mid_path = source.absolute().relative_to(Path(data_dir).absolute())
                     if use_suggested_path:
                         with global_env_lock:
-                            source_name = (
-                                source.name
-                                if source.name != "Images-Disc1"
-                                else source.parent.name
-                            )
+                            if source.name == "Images-Disc1":
+                                source_name = source.parent.name
+                            elif source.name.startswith("Sample"):
+                                source_name = f"{source.parent.name}_{source.name}"
+                            else:
+                                source_name = source.name
                             if environment.destination_registry.get(source_name):
                                 _default = environment.destination_registry[source_name]
                             else:
diff --git a/src/murfey/client/watchdir_multigrid.py b/src/murfey/client/watchdir_multigrid.py
@@ -51,6 +51,55 @@ def stop(self):
             self.thread.join()
         log.debug("MultigridDirWatcher thread stop completed")
 
+    def _handle_metadata(self, directory: Path, extra_directory: str):
+        self.notify(
+            directory,
+            extra_directory=extra_directory,
+            include_mid_path=False,
+            analyse=self._analyse,
+            limited=True,
+            tag="metadata",
+        )
+        self._seen_dirs.append(directory)
+
+    def _handle_fractions(self, directory: Path, first_loop: bool):
+        processing_started = False
+        for d02 in directory.glob("Images-Disc*"):
+            if d02 not in self._seen_dirs:
+                # If 'skip_existing_processing' is set, do not process for
+                # any data directories found on the first loop.
+                # This allows you to avoid triggering processing again if Murfey is restarted
+                self.notify(
+                    d02,
+                    include_mid_path=False,
+                    remove_files=True,
+                    analyse=(
+                        not (first_loop and self._skip_existing_processing)
+                        if self._analyse
+                        else False
+                    ),
+                    tag="fractions",
+                )
+                self._seen_dirs.append(d02)
+            processing_started = d02 in self._seen_dirs
+        if not processing_started:
+            if (
+                directory.is_dir()
+                and directory not in self._seen_dirs
+                and list(directory.iterdir())
+            ):
+                self.notify(
+                    directory,
+                    include_mid_path=False,
+                    analyse=(
+                        not (first_loop and self._skip_existing_processing)
+                        if self._analyse
+                        else False
+                    ),
+                    tag="fractions",
+                )
+                self._seen_dirs.append(directory)
+
     def _process(self):
         first_loop = True
         while not self._stopping:
@@ -75,53 +124,28 @@ def _process(self):
                         )
                         self._seen_dirs.append(d)
                 else:
-                    if d.is_dir() and d not in self._seen_dirs:
-                        self.notify(
-                            d,
-                            extra_directory=f"metadata_{d.name}",
-                            include_mid_path=False,
-                            analyse=self._analyse,
-                            limited=True,
-                            tag="metadata",
-                        )
-                        self._seen_dirs.append(d)
-                    processing_started = False
-                    for d02 in (d.parent.parent / d.name).glob("Images-Disc*"):
-                        if d02 not in self._seen_dirs:
-                            # If 'skip_existing_processing' is set, do not process for
-                            # any data directories found on the first loop.
-                            # This allows you to avoid triggering processing again if Murfey is restarted
-                            self.notify(
-                                d02,
-                                include_mid_path=False,
-                                remove_files=True,
-                                analyse=(
-                                    not (first_loop and self._skip_existing_processing)
-                                    if self._analyse
-                                    else False
-                                ),
-                                tag="fractions",
-                            )
-                            self._seen_dirs.append(d02)
-                        processing_started = d02 in self._seen_dirs
-                    if not processing_started:
-                        d02 = d.parent.parent / d.name
-                        if (
-                            d02.is_dir()
-                            and d02 not in self._seen_dirs
-                            and list((d.parent.parent / d.name).iterdir())
-                        ):
-                            self.notify(
-                                d02,
-                                include_mid_path=False,
-                                analyse=(
-                                    not (first_loop and self._skip_existing_processing)
-                                    if self._analyse
-                                    else False
-                                ),
-                                tag="fractions",
+                    # hack for tomo multigrid metadata structure
+                    sample_dirs = list(d.glob("Sample*"))
+                    if d.is_dir() and len(sample_dirs):
+                        for sample in sample_dirs:
+                            if len(list(sample.glob("*.mdoc"))):
+                                if sample not in self._seen_dirs:
+                                    self._handle_metadata(
+                                        sample,
+                                        extra_directory=f"metadata_{sample.parent.name}_{sample.name}",
+                                    )
+                                self._handle_fractions(
+                                    sample.parent.parent.parent
+                                    / f"{sample.parent.name}_{sample.name}",
+                                    first_loop,
+                                )
+
+                    else:
+                        if d.is_dir() and d not in self._seen_dirs:
+                            self._handle_metadata(
+                                d, extra_directory=f"metadata_{d.name}"
                             )
-                            self._seen_dirs.append(d02)
+                        self._handle_fractions(d.parent.parent / d.name, first_loop)
 
             if first_loop:
                 first_loop = False
diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py
@@ -2300,10 +2300,10 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
                 .where(db.ProcessingJob.recipe == "em-tomo-preprocess")
             ).one()
             if not _db.exec(
-                select(func.count(db.TomographyProcessingParameters.dcg_id)).where(
+                select(db.TomographyProcessingParameters.dcg_id).where(
                     db.TomographyProcessingParameters.dcg_id == collected_ids[0].id
                 )
-            ).one():
+            ).all():
                 params = db.TomographyProcessingParameters(
                     dcg_id=collected_ids[0].id,
                     pixel_size=float(message["pixel_size_on_image"]) * 10**10,
diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml
@@ -826,7 +826,7 @@ murfey.server.api.session_control.router:
         type: int
     methods:
       - DELETE
-  - path: /session_info/sessions/{session_id}/data_collection_groups
+  - path: /session_control/sessions/{session_id}/data_collection_groups
     function: get_dc_groups
     path_params:
       - name: session_id
