fix: handle resource downloads whose target directory got deleted

Author: paulmueller

CHANGELOG

@@ -1,3 +1,5 @@
+1.0.1
+ - fix: handle resource downloads whose target directory got deleted
 1.0.0
  - fix: do not attempt to updates non-existent eternal jobs on exit
  - enh: allow users to specify circle IDs as well as names during upload

dcoraid/download/job.py

@@ -30,6 +30,10 @@
 ]
 
 
+class InvalidDownloadDirectoryError(BaseException):
+    """Used for signaling a non-existent download directory"""
+
+
 class DownloadJob:
     def __init__(self, api, resource_id, download_path, condensed=False):
         """Wrapper for resource downloads
@@ -155,7 +159,13 @@ def get_dataset_dict(self):
         return self._dataset_dict
 
     def get_download_path(self):
-        """Return the final location to which the file is downloaded"""
+        """Return the final location to which the file is downloaded
+
+        Raises
+        ------
+        InvalidDownloadDirectoryError
+            When the target directory does not exist
+        """
         if self._download_path is None:
             if self._user_path.is_dir():
                 # Compute the resource path from the dataset dictionary
@@ -178,7 +188,7 @@ def get_download_path(self):
                 # user specified an actual file
                 self._download_path = self._user_path
             else:
-                raise ValueError(
+                raise InvalidDownloadDirectoryError(
                     f"The `download_path` passed in __init__ is invalid. "
                     f"Please make sure the target directory for "
                     f"{self._user_path} exists.")
@@ -309,7 +319,7 @@ def get_status(self):
                 data["bytes local"] = size_temp
             else:
                 data["bytes local"] = 0
-        except api_errors.APINotFoundError:
+        except (api_errors.APINotFoundError, InvalidDownloadDirectoryError):
             # The user likely tried to download the file from a different
             # host or an evil admin deleted a file.
             self.traceback = traceback.format_exc()
@@ -326,7 +336,11 @@ def get_status(self):
     def retry_download(self):
         """Retry downloading resources when an error occured"""
         if self.state in ["abort", "error"]:
+            # Create the target download directory
+            if self._user_path is not None:
+                self._user_path.parent.mkdir(parents=True, exist_ok=True)
             self.set_state("init")
+            self.traceback = None
         else:
             raise ValueError("Can only retry download in error state!")
 

dcoraid/gui/panel_downloads/widget_actions_download.py

@@ -51,7 +51,7 @@ def on_error(self):
                     + "<a href='https://github.com/DCOR-dev/DCOR-Aid/issues'>"
                     + "create an issue online</a>. Click on the button below "
                       "to see details required for fixing the problem.")
-        msg.setWindowTitle(f"Job {self.job.dataset_id[:5]} error")
+        msg.setWindowTitle(f"Job {self.job.job_id[:5]} error")
         msg.setDetailedText(self.job.traceback)
         msg.exec()
 

dcoraid/gui/panel_downloads/widget_download.py

@@ -231,23 +231,32 @@ def update_job_status(self):
         for row, job in enumerate(self.jobs):
             try:
                 status = job.get_status()
+                state = status["state"]
                 self.set_label_item(row, 0, job.job_id[:5])
-                try:
-                    title = get_download_title(job)
-                except BaseException:
-                    logger.error(traceback.format_exc())
-                    # Probably a connection error
-                    title = "-- error getting dataset title --"
-                self.set_label_item(row, 1, title)
-                self.set_label_item(row, 2, status["state"])
-                self.set_label_item(row, 3, job.get_progress_string())
-                self.set_label_item(row, 4, job.get_rate_string())
-                self.set_actions_item(row, 5, job)
-                if status["state"] == "done":
-                    self.on_download_finished(job.job_id)
             except BaseException:
+                logger.error(traceback.format_exc())
                 job.set_state("error")
                 job.traceback = traceback.format_exc()
+                state = "error"
+
+            try:
+                title = get_download_title(job)
+                progress = job.get_progress_string()
+                rate = job.get_rate_string()
+            except BaseException:
+                logger.error(traceback.format_exc())
+                # Probably a connection error
+                title = "-- error getting dataset title --"
+                progress = "unknown"
+                rate = "unknown"
+
+            self.set_label_item(row, 1, title)
+            self.set_label_item(row, 2, state)
+            self.set_label_item(row, 3, progress)
+            self.set_label_item(row, 4, rate)
+            self.set_actions_item(row, 5, job)
+            if state == "done":
+                self.on_download_finished(job.job_id)
 
             QtWidgets.QApplication.processEvents(
                 QtCore.QEventLoop.ProcessEventsFlag.AllEvents, 300)

tests/test_download_job.py

@@ -1,3 +1,4 @@
+import shutil
 import tempfile
 from unittest import mock
 import uuid
@@ -21,6 +22,47 @@ def test_initialize():
     assert dj.state == "init"
 
 
+def test_download_deleted_directory():
+    api = common.get_api()
+    td = tempfile.mkdtemp(prefix="test-download")
+    ds_dict = common.make_dataset_for_download()
+    dj = job.DownloadJob(api=api,
+                         resource_id=ds_dict["resources"][0]["id"],
+                         download_path=td)
+    # sanity check
+    assert dj.get_status()["bytes total"] == 899298
+    dj_state = dj.__getstate__()
+
+    # delete the job and its directory
+    path_dir = dj.path_dir
+    del dj
+    shutil.rmtree(path_dir)
+
+    # create a new job
+    dj2 = job.DownloadJob(api=api, **dj_state)
+
+    with pytest.raises(job.InvalidDownloadDirectoryError):
+        dj2.get_download_path()
+
+    assert np.isnan(dj2.get_status()["bytes total"])
+    assert dj2.traceback
+
+    # retry
+    dj2.retry_download()
+    assert dj2.get_status()["bytes total"] == 899298
+
+    # now attempt a resume-download
+    # create a truncated temporary file
+    dj2.task_download_resource()
+    dj2.task_verify_resource()
+    assert not dj2.traceback
+    assert dj2.start_time is not None
+    assert dj2.end_time is not None
+    assert dj2.path.exists()
+    assert dj2.file_bytes_downloaded == 899298
+    assert dj2.file_size == 899298
+
+
 def test_download_resume():
     api = common.get_api()
     td = tempfile.mkdtemp(prefix="test-download")