feat: more upload options

mapillary_tools/api_v4.py

@@ -162,18 +162,20 @@ def request_post(
     url: str,
     data: T.Any | None = None,
     json: dict | None = None,
+    disable_debug=False,
     **kwargs,
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
-    _log_debug_request(
-        "POST",
-        url,
-        json=json,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
+    if not disable_debug:
+        _log_debug_request(
+            "POST",
+            url,
+            json=json,
+            params=kwargs.get("params"),
+            headers=kwargs.get("headers"),
+            timeout=kwargs.get("timeout"),
+        )
 
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
@@ -193,25 +195,25 @@ def request_post(
             )
             return request_post(url, data=data, json=json, **kwargs)
 
-    _log_debug_response(resp)
+    if not disable_debug:
+        _log_debug_response(resp)
 
     return resp
 
 
 def request_get(
-    url: str,
-    params: dict | None = None,
-    **kwargs,
+    url: str, params: dict | None = None, disable_debug=False, **kwargs
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
-    _log_debug_request(
-        "GET",
-        url,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
+    if not disable_debug:
+        _log_debug_request(
+            "GET",
+            url,
+            params=kwargs.get("params"),
+            headers=kwargs.get("headers"),
+            timeout=kwargs.get("timeout"),
+        )
 
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
@@ -230,7 +232,8 @@ def request_get(
             )
             resp = request_get(url, params=params, **kwargs)
 
-    _log_debug_response(resp)
+    if not disable_debug:
+        _log_debug_response(resp)
 
     return resp
 
@@ -302,8 +305,7 @@ def fetch_organization(
 
 
 def fetch_user_or_me(
-    user_access_token: str,
-    user_id: int | str | None = None,
+    user_access_token: str, user_id: int | str | None = None
 ) -> requests.Response:
     if user_id is None:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
@@ -341,6 +343,7 @@ def log_event(action_type: ActionType, properties: dict) -> requests.Response:
             "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
         },
         timeout=REQUESTS_TIMEOUT,
+        disable_debug=True,
     )
     resp.raise_for_status()
     return resp

mapillary_tools/authenticate.py

@@ -87,8 +87,7 @@ def authenticate(
 
 
 def fetch_user_items(
-    user_name: str | None = None,
-    organization_key: str | None = None,
+    user_name: str | None = None, organization_key: str | None = None
 ) -> config.UserItem:
     """
     Read user information from the config file,

mapillary_tools/commands/upload.py

@@ -8,13 +8,13 @@
 
 class Command:
     name = "upload"
-    help = "upload images to Mapillary"
+    help = "Upload processed data to Mapillary"
 
     @staticmethod
     def add_common_upload_options(group):
         group.add_argument(
             "--user_name",
-            help="The Mapillary user account to upload to. If you only have one account authorized, it will upload to that account by default.",
+            help="The Mapillary user account to upload to.",
             required=False,
         )
         group.add_argument(
@@ -23,9 +23,31 @@ def add_common_upload_options(group):
             default=None,
             required=False,
         )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
         group.add_argument(
             "--dry_run",
-            help='Instead of uploading to the Mapillary server, simulate uploading to the local directory "mapillary_public_uploads" for debugging purposes.',
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
             action="store_true",
             default=False,
             required=False,
@@ -35,7 +57,7 @@ def add_basic_arguments(self, parser):
         group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
-            help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
             default=None,
             required=False,
         )

mapillary_tools/exif_write.py

@@ -42,7 +42,9 @@ def decimal_to_dms(
 
     def add_image_description(self, data: dict) -> None:
         """Add a dict to image description."""
-        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
+        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(
+            data, sort_keys=True, separators=(",", ":")
+        )
 
     def add_orientation(self, orientation: int) -> None:
         """Add image orientation to image."""

mapillary_tools/history.py

@@ -36,10 +36,21 @@ def history_desc_path(md5sum: str) -> Path:
     )
 
 
-def is_uploaded(md5sum: str) -> bool:
+def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]:
     if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
-        return False
-    return history_desc_path(md5sum).is_file()
+        return None
+
+    path = history_desc_path(md5sum)
+
+    if not path.is_file():
+        return None
+
+    with path.open("r") as fp:
+        try:
+            return json.load(fp)
+        except json.JSONDecodeError as ex:
+            LOG.error(f"Failed to read upload history {path}: {ex}")
+            return None
 
 
 def write_history(
@@ -53,10 +64,7 @@ def write_history(
     path = history_desc_path(md5sum)
     LOG.debug("Writing upload history: %s", path)
     path.resolve().parent.mkdir(parents=True, exist_ok=True)
-    history: dict[str, T.Any] = {
-        "params": params,
-        "summary": summary,
-    }
+    history: dict[str, T.Any] = {"params": params, "summary": summary}
     if metadatas is not None:
         history["descs"] = [
             DescriptionJSONSerializer.as_desc(metadata) for metadata in metadatas

mapillary_tools/mp4/construct_mp4_parser.py

@@ -567,7 +567,9 @@ def _new_cmap_without_boxes(
 # pyre-ignore[9]: pyre does not support recursive type SwitchMapType
 MP4_WITHOUT_STBL_CMAP: SwitchMapType = {
     # pyre-ignore[6]: pyre does not support recursive type SwitchMapType
-    b"moov": _new_cmap_without_boxes(CMAP[b"moov"], [b"stbl"]),
+    b"moov": _new_cmap_without_boxes(
+        CMAP[b"moov"], T.cast(T.Sequence[BoxType], [b"stbl"])
+    ),
 }
 
 # for parsing mp4 only

mapillary_tools/process_geotag_properties.py

@@ -227,6 +227,8 @@ def _show_stats(
     metadatas: T.Sequence[types.MetadataOrError],
     skipped_process_errors: set[T.Type[Exception]],
 ) -> None:
+    LOG.info("========== Process summary ==========")
+
     metadatas_by_filetype: dict[types.FileType, list[types.MetadataOrError]] = {}
     for metadata in metadatas:
         if isinstance(metadata, types.ImageMetadata):