solve base file missing

Author: MarcelGeo

mergin/client_pull.py

@@ -531,49 +531,63 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
     # list of FileToMerge instances, which consists of destination path and list of download items
     merge_files = []
     diff_files = []
-    # make sure we can update geodiff reference files (aka. basefiles) with diffs or
-    # download their full versions so we have them up-to-date for applying changes
     basefiles_to_patch = []  # list of tuples (relative path within project, list of diff files in temp dir to apply)
     pull_actions = []
     local_delta = mp.get_local_delta(tmp_dir.name)
+    # Converting local to PullActions
     for item in delta.items:
+        # find corresponding local delta item
         local_item = next((i for i in local_delta if i.path == item.path), None)
-        pull_action = mp.get_pull_action(item.change, local_item.change if local_item else None)
-        if not pull_action:
+        local_item_change = local_item.change if local_item else None
+
+        # compare server and local changes to decide what to do in pull
+        pull_action_type = mp.get_pull_action(item.change, local_item_change)
+        if not pull_action_type:
             continue  # no action needed
 
-        pull_actions.append(PullAction(pull_action, item, local_item))
-        if pull_action == PullActionType.APPLY_DIFF or (
-            pull_action == PullActionType.COPY_CONFLICT and item.change == DeltaChangeType.UPDATE_DIFF
+        pull_action = PullAction(pull_action_type, item, local_item)
+        if pull_action_type == PullActionType.APPLY_DIFF or (
+            pull_action_type == PullActionType.COPY_CONFLICT and item.change == DeltaChangeType.UPDATE_DIFF
         ):
+            basefile = mp.fpath_meta(item.path)
+            if not os.path.exists(basefile):
+                # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file
+                # or we removed it within previous pull because we failed to apply patch the older version for some reason).
+                # But it's not a problem - we will download the newest version and we're sorted.
+                mp.log.info(f"missing base file for {item.path} -> going to download it (version {server_version})")
+                items = get_download_items(item.path, item.size, server_version, tmp_dir.name)
+                dest_file_path = mp.fpath(item.path, tmp_dir.name)
+                merge_files.append(FileToMerge(dest_file_path, items))
+
+                # Force use COPY action to apply the new version instead of trying to apply diffs
+                # We are not able to get local changes anyway as base file is missing
+                pull_action.type = PullActionType.COPY_CONFLICT
+
             # if we have diff to apply, let's download the diff files
             # if we have conflict and diff update, download the diff files
-            if v2_pull:
+            elif v2_pull:
                 diff_files.extend(
                     [
                         DownloadDiffQueueItem(diff_item.id, os.path.join(tmp_dir.name, diff_item.id))
                         for diff_item in item.diffs
                     ]
                 )
+                basefiles_to_patch.append((item.path, [diff.id for diff in item.diffs]))
+
             else:
+                # fallback for diff files using v1 endpoint /raw
                 diff_merge_files = get_diff_merge_files(item, tmp_dir.name)
                 merge_files.extend(diff_merge_files)
+                basefiles_to_patch.append((item.path, [diff.id for diff in item.diffs]))
+
             # let's check the base file existence
-            basefile = mp.fpath_meta(item.path)
-            if not os.path.exists(basefile):
-                # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file
-                # or we removed it within previous pull because we failed to apply patch the older version for some reason).
-                # But it's not a problem - we will download the newest version and we're sorted.
-                mp.log.info(f"missing base file for {item.path} -> going to download it (version {server_version})")
-                items = get_download_items(item.path, item.size, server_version, tmp_dir.name)
-                dest_file_path = mp.fpath(item.path, tmp_dir.name)
-                merge_files.append(FileToMerge(dest_file_path, items))
-            basefiles_to_patch.append((item.path, [diff.id for diff in item.diffs]))
-        elif pull_action == PullActionType.COPY or pull_action == PullActionType.COPY_CONFLICT:
+        elif pull_action_type == PullActionType.COPY or pull_action_type == PullActionType.COPY_CONFLICT:
             # simply download the server version of the files
             dest_file_path = prepare_file_destination(tmp_dir.name, item.path)
             download_items = get_download_items(item.path, item.size, server_version, tmp_dir.name)
             merge_files.append(FileToMerge(dest_file_path, download_items))
+
+        pull_actions.append(pull_action)
         # Do nothing for DELETE actions
 
     # make a single list of items to download
@@ -693,7 +707,7 @@ def pull_project_finalize(job: PullJob):
         basefile = job.mp.fpath_meta(file_path)
         server_file = job.mp.fpath(file_path, job.tmp_dir.name)
 
-        shutil.copy(basefile, server_file)
+        job.mp.geodiff.make_copy_sqlite(basefile, server_file)
         diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs]
         patch_error = job.mp.apply_diffs(server_file, diffs)
         if patch_error:
@@ -709,6 +723,7 @@ def pull_project_finalize(job: PullJob):
             os.remove(basefile)
             raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile))
     conflicts = []
+    job.mp.log.info(f"--- applying pull actions {job.pull_actions}")
     try:
         if job.pull_actions:
             conflicts = job.mp.apply_pull_actions(job.pull_actions, job.tmp_dir.name, job.project_info, job.mc)

mergin/merginproject.py

@@ -633,7 +633,6 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaItem]:
                 # probably the database schema has been modified if geodiff cannot create changeset.
                 # we will need to do full upload of the file
                 pass
-                print(delta_item)
             result.append(delta_item)
 
         return result
@@ -732,9 +731,7 @@ def get_list_of_push_changes(self, push_changes):
                     pass
         return changes
 
-    def apply_pull_actions(
-        self, actions: List[PullAction], downloaded_files_directory: str, server_project: dict, mc
-    ) -> List[str]:
+    def apply_pull_actions(self, actions: List[PullAction], download_dir: str, server_project: dict, mc) -> List[str]:
         """
         Apply pull actions for files.
 
@@ -743,8 +740,8 @@ def apply_pull_actions(
 
         :param actions: list of pull actions
         :type actions: List[PullAction]
-        :param downloaded_files_directory: directory with downloaded files from server
-        :type downloaded_files_directory: str
+        :param download_dir: directory with downloaded files from server
+        :type download_dir: str
         :param server_project: project metadata from the server
         :type server_project: dict
         :param mc: mergin client
@@ -756,44 +753,42 @@ def apply_pull_actions(
 
         for action in actions:
             path = action.pull_delta_item.path
-            src = self.fpath(path, downloaded_files_directory)
-            dest = self.fpath(path)
+            server = self.fpath(path, download_dir)
+            live = self.fpath(path)
             basefile = self.fpath_meta(path)
             action_type = action.type
             pull_change = action.pull_delta_item.change
             local_change = action.local_delta_item.change if action.local_delta_item else None
             if action_type == PullActionType.COPY:
                 # simply copy the file from server
                 if is_versioned_file(path):
-                    self.geodiff.make_copy_sqlite(src, dest)
-                    self.geodiff.make_copy_sqlite(src, basefile)
+                    self.geodiff.make_copy_sqlite(server, live)
+                    self.geodiff.make_copy_sqlite(server, basefile)
                 else:
-                    shutil.copy(src, dest)
+                    shutil.copy(server, live)
 
             elif action_type == PullActionType.APPLY_DIFF:
                 # rebase needed only if both server and local changes are diffs
                 if pull_change == DeltaChangeType.UPDATE_DIFF and local_change == DeltaChangeType.UPDATE_DIFF:
-                    conflict = self.update_with_rebase(
-                        path, src, dest, basefile, downloaded_files_directory, mc.username()
-                    )
+                    conflict = self.update_with_rebase(path, server, live, basefile, download_dir, mc.username())
                     if conflict:
                         conflicts.append(conflict)
                 else:
                     # no rebase needed, just apply the diff
-                    self.update_without_rebase(path, src, dest, basefile, downloaded_files_directory)
+                    self.update_without_rebase(path, server, live, basefile, download_dir)
 
             elif action_type == PullActionType.COPY_CONFLICT and not prevent_conflicted_copy(path, mc, server_project):
                 conflict = self.create_conflicted_copy(path, mc.username())
                 conflicts.append(conflict)
                 if self.is_versioned_file(path):
-                    self.geodiff.make_copy_sqlite(src, dest)
-                    self.geodiff.make_copy_sqlite(src, basefile)
+                    self.geodiff.make_copy_sqlite(server, live)
+                    self.geodiff.make_copy_sqlite(server, basefile)
                 else:
-                    shutil.copy(src, dest)
+                    shutil.copy(server, live)
 
             elif action_type == PullActionType.DELETE:
                 # remove local file
-                os.remove(dest)
+                os.remove(live)
                 if self.is_versioned_file(path):
                     os.remove(basefile)
 

mergin/test/test_client.py

@@ -280,10 +280,10 @@ def test_create_remote_project_from_local(mc):
     project_info = mc.project_info_v2(project_info.get("id"))
     assert project_info.version == source_mp.version()
     assert project_info.name == test_project
-    assert project_info.namespace == API_USER
+    assert project_info.workspace.name == API_USER
     assert project_info.id == source_mp.project_id()
 
-    version = mc.project_version_info(project_info.get("id"), "v1")
+    version = mc.project_version_info(project_info.id, "v1")
     assert version["name"] == "v1"
     assert any(f for f in version["changes"]["added"] if f["path"] == "test.qgs")
 
@@ -690,7 +690,9 @@ def test_missing_basefile_pull(mc):
 
     # try to sync again  -- it should not crash
     mc.pull_project(project_dir)
+    assert os.path.exists(os.path.join(project_dir, ".mergin", "base.gpkg"))
     mc.push_project(project_dir)
+    # check if base file exists again
 
 
 def test_empty_file_in_subdir(mc):
@@ -3068,7 +3070,7 @@ def test_pull_project(mc: MerginClient, mc2: MerginClient):
     assert mp_to_pull.version() == mp.version()
     assert mp_to_pull.project_id() == mp.project_id()
     assert len(project_info.get("files")) == len(mp.files())
-    delta_items = mp.get_pull_delta({"files": mp_to_pull.files(), "version": mp_to_pull.version()})
-    for item in [item for item in delta_items if item.change == DeltaChangeType.CREATE]:
+    delta = mp.get_pull_delta({"files": mp_to_pull.files(), "version": mp_to_pull.version()})
+    for item in [item for item in delta.items if item.change == DeltaChangeType.CREATE]:
         assert os.path.exists(mp_to_pull.fpath(item.path))
     assert os.path.exists(mp_to_pull.fpath_meta("base.gpkg"))