Merge pull request #14 from MITLibraries/misc-refactoring

get id from handle and misc refactoring

Author: ehanson8

.travis.yml

@@ -1,6 +1,6 @@
 language: python
 python:
-  - "3.7"
+  - "3.8"
 install:
   - pipenv install --dev
 script:

Pipfile

@@ -10,12 +10,12 @@ requests-mock = "*"
 [packages]
 requests = "*"
 structlog = "*"
-attr = "*"
+attrs = "*"
 click = "*"
 lxml = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"
 
 [scripts]
 dsaps = "python -c \"from dsaps.cli import main; main()\""

dsaps/cli.py

@@ -24,6 +24,8 @@
 @click.pass_context
 def main(ctx, url, email, password):
     ctx.obj = {}
+    if os.path.isdir('logs') is False:
+        os.mkdir('logs')
     dt = datetime.datetime.utcnow().isoformat(timespec='seconds')
     log_suffix = f'{dt}.log'
     structlog.configure(processors=[
@@ -47,31 +49,6 @@ def main(ctx, url, email, password):
     ctx.obj['start_time'] = start_time
 
 
-@click.group()
-def aux():
-    pass
-
-
-@main.command()
-@click.option('-f', '--field', prompt='Enter the field to be searched',
-              help='The field to search.')
-@click.option('-s', '--string', prompt='Enter the string',
-              help='The field to search.')
-@click.option('-t', '--search_type', prompt='Enter the type of search',
-              help='The type of search.',
-              type=click.Choice(['exists', 'doesnt_exist', 'equals',
-                                 'not_equals', 'contains', 'doesnt_contain']),
-              default='contains')
-@click.pass_context
-def search(ctx, field, string, search_type):
-    # Temp function for testing
-    client = ctx.obj['client']
-    start_time = ctx.obj['start_time']
-    item_links = client.filtered_item_search(field, string, search_type)
-    logger.info(item_links)
-    models.elapsed_time(start_time, 'Elapsed time')
-
-
 @main.command()
 @click.option('-c', '--comm_handle', prompt='Enter the community handle',
               help='The handle of the community in which to create the ,'
@@ -111,7 +88,7 @@ def newcoll(ctx, comm_handle, coll_name, metadata, file_path, file_type,
     models.elapsed_time(start_time, 'Total runtime:')
 
 
-@aux.command()
+@main.command()
 @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
               help='The path of the CSV file of metadata.')
 @click.option('-f', '--file_path', prompt='Enter the path',
@@ -150,7 +127,7 @@ def reconcile(metadata_csv, file_path, file_type):
     models.create_csv_from_list(metadata_matches, 'metadata_matches.csv')
 
 
-@aux.command()
+@main.command()
 @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
               help='The path of the CSV file of metadata.')
 def metadatajson(metadata_csv):
@@ -190,7 +167,5 @@ def metadatajson(metadata_csv):
         json.dump(metadata_group, f)
 
 
-cli = click.CommandCollection(sources=[main, aux])
-
 if __name__ == '__main__':
-    cli()
+    main()

dsaps/models.py

@@ -1,3 +1,4 @@
+import collections
 import csv
 import datetime
 from functools import partial
@@ -77,6 +78,13 @@ def filtered_item_search(self, key, string, query_type,
             offset = offset + 200
         return item_links
 
+    def get_id_from_handle(self, handle):
+        """Retrieves UUID for an object based on its handle."""
+        endpoint = f'{self.url}/handle/{handle}'
+        rec_obj = requests.get(endpoint, headers=self.header,
+                               cookies=self.cookies).json()
+        return rec_obj['uuid']
+
     def post_coll_to_comm(self, comm_handle, coll_name):
         """Posts a collection to a specified community."""
         endpoint = f'{self.url}/handle/{comm_handle}'
@@ -117,22 +125,29 @@ def post_items_to_coll(self, coll_id, coll_metadata, file_dict,
 
     def post_bitstreams_to_item(self, item_id, file_identifier, file_dict,
                                 ingest_type):
-        """Posts bitstreams to a specified item."""
-        for k, v in file_dict.items():
-            if k.startswith(file_identifier):
-                bitstream = file_dict[k]
-                file_name = os.path.basename(bitstream)
-                if ingest_type == 'local':
-                    data = open(bitstream, 'rb')
-                elif ingest_type == 'remote':
-                    data = requests.get(bitstream)
-                endpoint = (f'{self.url}/items/{item_id}'
-                            + f'/bitstreams?name={file_name}')
-                header_upload = {'accept': 'application/json'}
-                bit_id = requests.post(endpoint, headers=header_upload,
-                                       cookies=self.cookies, data=data).json()
-                bit_id = bit_id['uuid']
-                yield bit_id
+        """Post a sorted set of bitstreams to a specified item."""
+        file_dict = collections.OrderedDict(sorted(file_dict.items()))
+        for bitstream, v in file_dict.items():
+            bit_id = self.post_bitstream(item_id, file_identifier, file_dict,
+                                         ingest_type, bitstream)
+            yield bit_id
+
+    def post_bitstream(self, item_id, file_identifier, file_dict, ingest_type,
+                       bitstream):
+        """Post a bitstream to a specified item."""
+        bitstream_path = file_dict[bitstream]
+        file_name = os.path.basename(bitstream_path)
+        if ingest_type == 'local':
+            data = open(bitstream_path, 'rb')
+        elif ingest_type == 'remote':
+            data = requests.get(bitstream_path)
+        endpoint = (f'{self.url}/items/{item_id}'
+                    + f'/bitstreams?name={file_name}')
+        header_upload = {'accept': 'application/json'}
+        bit_id = requests.post(endpoint, headers=header_upload,
+                               cookies=self.cookies, data=data).json()
+        bit_id = bit_id['uuid']
+        return bit_id
 
     def _pop_inst(self, class_type, rec_obj):
         """Populate class instance with data from record."""
@@ -192,7 +207,7 @@ def build_file_dict_remote(directory_url, file_type, file_dict):
     """Build list of files in a remote directory."""
     response = requests.get(directory_url)
     links = html.fromstring(response.content).iterlinks()
-    for link in [l for l in links if l[2].endswith(file_type)]:
+    for link in [i for i in links if i[2].endswith(file_type)]:
         file_identifier = link[2].replace(f'.{file_type}', '')
         file_dict[file_identifier] = f'{directory_url}{link[2]}'
     return file_dict
@@ -213,24 +228,29 @@ def elapsed_time(start_time, label):
     logger.info(f'{label} : {td}')
 
 
-def metadata_csv(row, key, field, language=None):
-    """Create metadata element from CSV."""
-    value = row[field]
-    if language is not None:
-        metadata_elem = {'key': key, 'language': language, 'value':
-                         value}
-    else:
-        metadata_elem = {'key': key, 'value': value}
-    return metadata_elem
+def metadata_elems_from_row(row, key, field, language=None, delimiter=''):
+    """Create a metadata element from a CSV row."""
+    metadata_elems = []
+    if row[field] != '':
+        if delimiter:
+            values = row[field].split(delimiter)
+        else:
+            values = [row[field]]
+        for value in values:
+            metadata_elem = {'key': key, 'language': language, 'value':
+                             value}
+            metadata_elems.append({k: v for k, v in metadata_elem.items()
+                                  if v is not None})
+    return metadata_elems
 
 
 def create_metadata_rec(mapping_dict, row, metadata_rec):
-    """Create metadata record from CSV."""
+    """Create metadata record from a series of metadata elements."""
     for k, v in mapping_dict.items():
-        if len(v) == 2:
-            metadata_elem = metadata_csv(row, k, v[0], v[1])
+        if len(v) == 3:
+            metadata_elems = metadata_elems_from_row(row, k, v[0], v[1], v[2])
         else:
-            metadata_elem = metadata_csv(row, k, v[0])
-        if metadata_elem['value'] != '':
+            metadata_elems = metadata_elems_from_row(row, k, v[0])
+        for metadata_elem in metadata_elems:
             metadata_rec.append(metadata_elem)
     return metadata_rec

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 setup(
-    name='DSpace API Python Scripts',
+    name='dsaps',
     version='1.0.0',
     description='',
     packages=find_packages(exclude=['tests']),
@@ -10,7 +10,7 @@
     install_requires=[
         'requests',
         'structlog',
-        'attr',
+        'attrs',
         'click',
         'lxml',
     ],
@@ -19,5 +19,5 @@
             'dsaps=dsaps.cli:main',
         ]
     },
-    python_requires='>=3.7.1',
+    python_requires='>=3.8',
 )