speed up validation by resuing validators (#777)

* improve performance with validator instances

* fix

Author: ptpt

mapillary_tools/authenticate.py

@@ -165,7 +165,7 @@ def _prompt(message: str) -> str:
 
 def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, config.UserItemSchema)
+        config.UserItemSchemaValidator.validate(user_items)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"

mapillary_tools/config.py

@@ -6,6 +6,8 @@
 import typing as T
 from typing import TypedDict
 
+import jsonschema
+
 if sys.version_info >= (3, 11):
     from typing import Required
 else:
@@ -50,6 +52,9 @@ class UserItem(TypedDict, total=False):
 }
 
 
+UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)

mapillary_tools/geotag/options.py

@@ -173,8 +173,11 @@ class InterpolationOption:
 }
 
 
+SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
+
+
 def validate_option(instance):
-    jsonschema.validate(instance=instance, schema=SourceOptionSchema)
+    SourceOptionSchemaValidator.validate(instance=instance)
 
 
 if __name__ == "__main__":

mapillary_tools/process_geotag_properties.py

@@ -304,19 +304,12 @@ def _validate_metadatas(
     # TypeError: __init__() missing 3 required positional arguments: 'image_time', 'gpx_start_time', and 'gpx_end_time'
     # See https://stackoverflow.com/a/61432070
     good_metadatas, error_metadatas = types.separate_errors(metadatas)
-    map_results = utils.mp_map_maybe(
-        validate_and_fail_metadata,
-        T.cast(T.Iterable[types.Metadata], good_metadatas),
-        num_processes=num_processes,
-    )
 
     validated_metadatas = list(
-        tqdm(
-            map_results,
-            desc="Validating metadatas",
-            unit="metadata",
-            disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-            total=len(good_metadatas),
+        utils.mp_map_maybe(
+            validate_and_fail_metadata,
+            T.cast(T.Iterable[types.Metadata], good_metadatas),
+            num_processes=num_processes,
         )
     )
 

mapillary_tools/serializer/description.py

@@ -259,6 +259,11 @@ def _merge_schema(*schemas: dict) -> dict:
 )
 
 
+ImageDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
+    ImageDescriptionFileSchema
+)
+
+
 VideoDescriptionFileSchema = _merge_schema(
     VideoDescriptionSchema,
     {
@@ -295,6 +300,11 @@ def _merge_schema(*schemas: dict) -> dict:
 )
 
 
+VideoDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
+    VideoDescriptionFileSchema
+)
+
+
 ImageVideoDescriptionFileSchema = {
     "oneOf": [VideoDescriptionFileSchema, ImageDescriptionFileSchema]
 }
@@ -520,7 +530,7 @@ def parse_capture_time(time: str) -> datetime.datetime:
 
 def validate_image_desc(desc: T.Any) -> None:
     try:
-        jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
+        ImageDescriptionFileSchemaValidator.validate(desc)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
         raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
@@ -533,7 +543,7 @@ def validate_image_desc(desc: T.Any) -> None:
 
 def validate_video_desc(desc: T.Any) -> None:
     try:
-        jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
+        VideoDescriptionFileSchemaValidator.validate(desc)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
         raise exceptions.MapillaryMetadataValidationError(ex.message) from ex

mapillary_tools/upload.py

@@ -10,7 +10,6 @@
 from pathlib import Path
 
 import humanize
-import jsonschema
 import requests
 from tqdm import tqdm
 
@@ -57,7 +56,7 @@ def upload(
 
     metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
 
-    jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
+    config.UserItemSchemaValidator.validate(user_items)
 
     # Setup the emitter -- the order matters here