Introduce UploadChanges class and split method

Author: harminius

mergin/client_push.py

@@ -83,6 +83,38 @@ def upload_blocking(self, mc, mp):
                 raise ClientError("Mismatch between uploaded file chunk {} and local one".format(self.chunk_id))
 
 
+class UploadChanges:
+    def __init__(self, added=None, updated=None, removed=None):
+        self.added = added or []
+        self.updated = updated or []
+        self.removed = removed or []
+        self.renamed = []
+
+    def is_empty(self):
+        return not (self.added or self.updated or self.removed or self.renamed)
+
+    def split(self):
+        blocking = UploadChanges()
+        non_blocking = UploadChanges()
+
+        for file in self.added:
+            target = blocking if is_qgis_file(file["path"]) or is_versioned_file(file["path"]) else non_blocking
+            target.added.append(file)
+
+        for file in self.updated:
+            blocking.updated.append(file)
+
+        for file in self.removed:
+            blocking.removed.append(file)
+
+        result = {}
+        if not blocking.is_empty():
+            result["blocking"] = blocking
+        if not non_blocking.is_empty():
+            result["non_blocking"] = non_blocking
+        return result
+
+
 def push_project_async(mc, directory):
     """Starts push of a project and returns pending upload job"""
 
@@ -124,21 +156,22 @@ def push_project_async(mc, directory):
     changes = mp.get_push_changes()
     changes = filter_changes(mc, project_info, changes)
 
-    blocking_changes, non_blocking_changes = split_changes(changes)
+    blocking_changes, non_blocking_changes = changes.split()
 
     blocking_job = (
         _prepare_upload_job(mp, mc, project_path, local_version, blocking_changes)
         if any(len(v) for v in blocking_changes.values())
         else None
     )
     non_blocking_job = (
-        _prepare_upload_job(mp, mc, project_path, local_version,  non_blocking_changes)
+        _prepare_upload_job(mp, mc, project_path, local_version, non_blocking_changes)
         if any(len(v) for v in non_blocking_changes.values())
         else None
     )
 
     return blocking_job, non_blocking_job
 
+
 def _prepare_upload_job(mp, mc, project_path, local_version, changes):
     mp.log.debug("push changes:\n" + pprint.pformat(changes))
 
@@ -226,6 +259,7 @@ def _prepare_upload_job(mp, mc, project_path, local_version, changes):
         job.futures.append(future)
     return job
 
+
 def push_project_wait(job):
     """blocks until all upload tasks are finished"""
 
@@ -351,25 +385,3 @@ def remove_diff_files(job) -> None:
             if os.path.exists(diff_file):
                 os.remove(diff_file)
 
-
-def split_changes(changes):
-    """Split changes into blocking and non-blocking.
-
-    Blocking criteria:
-     - any updated files
-     - any removed files
-     - added files that are .gpkg or .qgz (.ggs)
-    """
-    blocking = non_blocking = {"added": [], "updated": [], "removed": [], "renamed": []}
-
-    blocking["updated"] = changes["updated"]
-    blocking["removed"] = changes["removed"]
-    blocking["renamed"] = changes["renamed"]
-
-    for f in changes["added"]:
-        if is_qgis_file(f["path"]) or is_versioned_file(f["path"]):
-            blocking["added"].append(f)
-        else:
-            non_blocking["added"].append(f)
-
-    return blocking, non_blocking

mergin/common.py

@@ -1,7 +1,7 @@
 import os
 from enum import Enum
 
-CHUNK_SIZE = 10 * 1024 * 1024
+CHUNK_SIZE = 100 * 1024 * 1024
 
 # there is an upper limit for chunk size on server, ideally should be requested from there once implemented
 UPLOAD_CHUNK_SIZE = 10 * 1024 * 1024

mergin/editor.py

@@ -1,6 +1,7 @@
 from itertools import filterfalse
 from typing import Callable, Dict, List
 
+from .client_push import UploadChanges
 from .utils import is_mergin_config, is_qgis_file, is_versioned_file
 
 EDITOR_ROLE_NAME = "editor"
@@ -24,7 +25,7 @@ def is_editor_enabled(mc, project_info: dict) -> bool:
     return server_support and project_role == EDITOR_ROLE_NAME
 
 
-def _apply_editor_filters(changes: Dict[str, List[dict]]) -> Dict[str, List[dict]]:
+def _apply_editor_filters(changes: UploadChanges) -> UploadChanges:
     """
     Applies editor-specific filters to the changes dictionary, removing any changes to files that are not in the editor's list of allowed files.
 
@@ -36,11 +37,11 @@ def _apply_editor_filters(changes: Dict[str, List[dict]]) -> Dict[str, List[dict
     """
     updated = changes.get("updated", [])
 
-    changes["updated"] = list(filterfalse(_disallowed_changes, updated))
+    changes.updated = list(filterfalse(_disallowed_changes, updated))
     return changes
 
 
-def filter_changes(mc, project_info: dict, changes: Dict[str, List[dict]]) -> Dict[str, List[dict]]:
+def filter_changes(mc, project_info: dict, changes: UploadChanges) -> UploadChanges:
     """
     Filters the given changes dictionary based on the editor's enabled state.
 

mergin/merginproject.py

@@ -9,6 +9,7 @@
 from datetime import datetime
 from dateutil.tz import tzlocal
 
+from .client_push import UploadChanges
 from .editor import prevent_conflicted_copy
 
 from .common import UPLOAD_CHUNK_SIZE, InvalidProject, ClientError
@@ -314,23 +315,13 @@ def inspect_files(self):
                 )
         return files_meta
 
-    def compare_file_sets(self, origin, current):
+    def compare_file_sets(self, origin, current) -> UploadChanges:
         """
-        Helper function to calculate difference between two sets of files metadata using file names and checksums.
+        Calculate difference between two sets of file metadata using file names and checksums.
 
-        :Example:
-
-        >>> origin = [{'checksum': '08b0e8caddafe74bf5c11a45f65cedf974210fed', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}]
-        >>> current = [{'checksum': 'c9a4fd2afd513a97aba19d450396a4c9df8b2ba4', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}]
-        >>> self.compare_file_sets(origin, current)
-        {"added": [{'checksum': 'c9a4fd2afd513a97aba19d450396a4c9df8b2ba4', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}], "removed": [[{'checksum': '08b0e8caddafe74bf5c11a45f65cedf974210fed', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}]], "renamed": [], "updated": []}
-
-        :param origin: origin set of files metadata
-        :type origin: list[dict]
-        :param current: current set of files metadata to be compared against origin
-        :type current: list[dict]
-        :returns: changes between two sets with change type
-        :rtype: dict[str, list[dict]]'
+        :param origin: List of original file metadata
+        :param current: List of current file metadata
+        :return: UploadChanges instance with added, updated, removed
         """
         origin_map = {f["path"]: f for f in origin}
         current_map = {f["path"]: f for f in current}
@@ -347,7 +338,12 @@ def compare_file_sets(self, origin, current):
             f["origin_checksum"] = origin_map[path]["checksum"]
             updated.append(f)
 
-        return {"renamed": [], "added": added, "removed": removed, "updated": updated}
+        return UploadChanges(
+            added=added,
+            updated=updated,
+            removed=removed,
+        )
+
 
     def get_pull_changes(self, server_files):
         """
@@ -405,7 +401,7 @@ def get_pull_changes(self, server_files):
         changes["updated"] = [f for f in changes["updated"] if f not in not_updated]
         return changes
 
-    def get_push_changes(self):
+    def get_push_changes(self) -> UploadChanges:
         """
         Calculate changes needed to be pushed to server.
 
@@ -427,9 +423,9 @@ def get_push_changes(self):
                 file["checksum"] = checksum
             file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(file["size"] / UPLOAD_CHUNK_SIZE))]
 
-        # need to check for for real changes in geodiff files using geodiff tool (comparing checksum is not enough)
+        # need to check for real changes in geodiff files using geodiff tool (comparing checksum is not enough)
         not_updated = []
-        for file in changes["updated"]:
+        for file in changes.updated:
             path = file["path"]
             if not self.is_versioned_file(path):
                 continue
@@ -463,7 +459,7 @@ def get_push_changes(self):
                 # we will need to do full upload of the file
                 pass
 
-        changes["updated"] = [f for f in changes["updated"] if f not in not_updated]
+        changes.updated = [f for f in changes.updated if f not in not_updated]
         return changes
 
     def copy_versioned_file_for_upload(self, f, tmp_dir):