Upload files in parallel

Author: Laurynas Butkus

convertapi/__init__.py

@@ -15,5 +15,6 @@
 conversion_timeout_delta = 10
 upload_timeout = 600
 download_timeout = 600
+max_parallel_uploads = 10
 
 client = Client()

convertapi/task.py

@@ -1,6 +1,6 @@
 import convertapi
 
-from convertapi import file_param, format_detector
+from convertapi import file_param, format_detector, utils
 from .result import Result
 
 DEFAULT_URL_FORMAT = 'url'
@@ -34,9 +34,11 @@ def __normalize_params(self):
             if k == 'File':
                 params[k] = file_param.build(v)
             elif k == 'Files':
-                for idx, val in enumerate(v):
+                results = utils.map_in_parallel(file_param.build, v, convertapi.max_parallel_uploads)
+
+                for idx, val in enumerate(results):
                     key = '%s[%i]' % (k, idx)
-                    params[key] = file_param.build(val)
+                    params[key] = val
             else:
                 params[k] = v
 

convertapi/utils.py

@@ -0,0 +1,9 @@
+import multiprocessing
+
+def map_in_parallel(f, values, pool_size):
+    pool = multiprocessing.Pool(pool_size)
+    results = pool.map_async(f, values)
+    pool.close()
+    pool.join()
+
+    return results.get()

tests/test_convertapi.py

@@ -35,5 +35,4 @@ def test_upload_io(self):
 	def test_zip_files(self):
 		files = ['examples/files/test.docx', 'examples/files/test.docx']
 		result = convertapi.convert('zip', { 'Files': files })
-		print result.file.url
 		assert result.conversion_cost > 0