reconcile refactor

Author: ehanson8

dsaps/cli.py

@@ -9,7 +9,7 @@
 import click
 import structlog
 
-from dsaps import models
+from dsaps import models, workflows
 
 logger = structlog.get_logger()
 
@@ -96,35 +96,7 @@ def newcoll(ctx, comm_handle, coll_name, metadata, file_path, file_type,
 @click.option('-t', '--file_type', prompt='Enter the file type',
               help='The file type to be uploaded.')
 def reconcile(metadata_csv, file_path, file_type):
-    if file_path.startswith('http'):
-        file_dict = models.build_file_dict_remote(file_path, file_type, {})
-    else:
-        files = glob.glob(f'{file_path}/**/*.{file_type}', recursive=True)
-        for file in files:
-            file_name = os.path.splitext(os.path.basename(file))[0]
-            file_dict[file_name] = file
-    metadata_ids = []
-    with open(metadata_csv) as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            value = row['file_identifier']
-            metadata_ids.append(value)
-    file_matches = []
-    file_ids = []
-    for file_id, v in file_dict.items():
-        file_ids.append(file_id)
-        for metadata_id in [m for m in metadata_ids if file_id == m]:
-            file_matches.append(file_id)
-    metadata_matches = []
-    for metadata_id in metadata_ids:
-        for file_id in file_dict:
-            if file_id == metadata_id:
-                metadata_matches.append(metadata_id)
-    no_files = set(metadata_ids) - set(metadata_matches)
-    no_metadata = set(file_ids) - set(file_matches)
-    models.create_csv_from_list(no_metadata, 'no_metadata.csv')
-    models.create_csv_from_list(no_files, 'no_files.csv')
-    models.create_csv_from_list(metadata_matches, 'metadata_matches.csv')
+    workflows.reconcile_files_and_metadata(metadata_csv, file_path, file_type)
 
 
 @main.command()

dsaps/workflows.py

@@ -0,0 +1,79 @@
+import csv
+import glob
+import os
+
+from dsaps import models
+
+
+def create_file_dict_and_list(file_path, file_type):
+    """Creates a dict of file IDs and file paths and a list of file IDs."""
+    if file_path.startswith('http'):
+        file_dict = models.build_file_dict_remote(file_path, file_type, {})
+    else:
+        files = glob.glob(f'{file_path}/**/*.{file_type}', recursive=True)
+        file_dict = {}
+        file_ids = []
+        for file in files:
+            file_name = os.path.splitext(os.path.basename(file))[0]
+            file_dict[file_name] = file
+            file_ids.append(file_name)
+    return file_dict, file_ids
+
+
+def create_metadata_id_list(metadata_csv):
+    """Creates a list of IDs from a metadata CSV"""
+    metadata_ids = []
+    with open(metadata_csv) as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            value = row['file_identifier']
+            metadata_ids.append(value)
+    return metadata_ids
+
+
+def match_files_to_metadata(file_dict, file_ids, metadata_ids):
+    """Creates a list of files matched to metadata records."""
+    file_matches = []
+    for file_id, v in file_dict.items():
+        for metadata_id in [m for m in metadata_ids
+                            if file_id.startswith(m)]:
+            file_matches.append(file_id)
+    return file_matches
+
+
+def match_metadata_to_files(file_dict, metadata_ids):
+    """Creates a list of metadata records matched to files."""
+    metadata_matches = []
+    for metadata_id in metadata_ids:
+        for file_id in file_dict:
+            if file_id.startswith(metadata_id):
+                metadata_matches.append(metadata_id)
+    return metadata_matches
+
+
+def reconcile_files_and_metadata(metadata_csv, file_path, file_type):
+    """Runs a reconciliation of files and metadata."""
+    file_dict, file_ids = create_file_dict_and_list(file_path, file_type)
+    metadata_ids = create_metadata_id_list(metadata_csv)
+    metadata_matches = match_metadata_to_files(file_dict, metadata_ids)
+    file_matches = match_files_to_metadata(file_dict, file_ids, metadata_ids)
+    no_files = set(metadata_ids) - set(metadata_matches)
+    no_metadata = set(file_ids) - set(file_matches)
+    models.create_csv_from_list(no_metadata, 'no_metadata')
+    models.create_csv_from_list(no_files, 'no_files')
+    models.create_csv_from_list(metadata_matches, 'metadata_matches')
+    update_metadata_csv(metadata_csv, metadata_matches)
+
+
+def update_metadata_csv(metadata_csv, metadata_matches):
+    """Creates an updated CSV of metadata records with matching files."""
+    with open(metadata_csv) as csvfile:
+        reader = csv.DictReader(csvfile)
+        upd_md_file_name = f'updated-{os.path.basename(metadata_csv)}'
+        with open(f'{upd_md_file_name}', 'w') as updated_csv:
+            writer = csv.DictWriter(updated_csv, fieldnames=reader.fieldnames)
+            writer.writeheader()
+            csvfile.seek(0)
+            for row in reader:
+                if row['file_identifier'] in metadata_matches:
+                    writer.writerow(row)

tests/conftest.py

@@ -1,3 +1,5 @@
+import csv
+
 from click.testing import CliRunner
 import pytest
 import requests_mock
@@ -37,10 +39,10 @@ def ds_mock():
         item_json = {'uuid': 'a1b2', 'handle': '1111.1/1111'}
         m.post('mock://example.com/collections/789/items', json=item_json)
         b_json_1 = {'uuid': 'c3d4'}
-        url_1 = 'mock://example.com/items/a1b2/bitstreams?name=123_1.pdf'
+        url_1 = 'mock://example.com/items/a1b2/bitstreams?name=test_01.pdf'
         m.post(url_1, json=b_json_1)
         b_json_2 = {'uuid': 'e5f6'}
-        url_2 = 'mock://example.com/items/a1b2/bitstreams?name=123_2.pdf'
+        url_2 = 'mock://example.com/items/a1b2/bitstreams?name=test_02.pdf'
         m.post(url_2, json=b_json_2)
         yield m
 
@@ -51,19 +53,18 @@ def runner():
 
 
 @pytest.fixture(autouse=True)
-def sample_content_1(tmp_path):
-    content = 'test'
-    dir = tmp_path / 'sub'
-    dir.mkdir()
-    sample_content = dir / '123_1.pdf'
-    sample_content.write_text(content)
-    return sample_content
-
-
-@pytest.fixture(autouse=True)
-def sample_content_2(tmp_path):
-    content = 'test'
-    dir = tmp_path / 'sub'
-    sample_content = dir / '123_2.pdf'
-    sample_content.write_text(content)
-    return sample_content
+def sample_files_dir(tmp_path):
+    sample_files_dir = tmp_path / 'files'
+    sample_files_dir.mkdir()
+    with open(f'{sample_files_dir}/test_01.pdf', 'w'):
+        pass
+    with open(f'{sample_files_dir}/test_02.pdf', 'w'):
+        pass
+    with open(f'{sample_files_dir}/best_01.pdf', 'w'):
+        pass
+    with open(f'{sample_files_dir}/metadata.csv', 'w') as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(['uri'] + ['title'] + ['file_identifier'])
+        writer.writerow(['/repo/0/ao/123'] + ['Test Item'] + ['test'])
+        writer.writerow(['/repo/0/ao/456'] + ['Tast Item'] + ['tast'])
+    return str(sample_files_dir)

tests/test_cli.py

@@ -0,0 +1,30 @@
+import csv
+import os
+import requests_mock
+
+from dsaps.cli import main
+
+
+def test_reconcile(runner):
+    """Test reconcile command."""
+    with requests_mock.Mocker() as m:
+        with runner.isolated_filesystem():
+            os.mkdir('files')
+            with open('metadata.csv', 'w') as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow(['uri'] + ['title'] + ['file_identifier'])
+                writer.writerow(['/repo/0/ao/123'] + ['Test Item'] + ['test'])
+            cookies = {'JSESSIONID': '11111111'}
+            user_json = {'fullname': 'User Name'}
+            m.post('mock://example.com/login', cookies=cookies)
+            m.get('mock://example.com/status', json=user_json)
+            result = runner.invoke(main,
+                                   ['--url', 'mock://example.com/',
+                                    '--email', 'test@test.mock',
+                                    '--password', '1234',
+                                    'reconcile',
+                                    '--metadata_csv', 'metadata.csv',
+                                    '--file_path', 'files',
+                                    '--file_type', 'pdf'
+                                    ])
+    assert result.exit_code == 0

tests/test_models.py

@@ -45,35 +45,51 @@ def test_post_coll_to_comm(client):
     assert coll_id == '5678'
 
 
-def test_post_items_to_coll(client, sample_content_1):
+def test_post_items_to_coll(client, sample_files_dir):
     """Test post_items_to_coll method."""
     coll_metadata = [{"metadata": [
                      {"key": "file_identifier",
-                      "value": "123"},
+                      "value": "test"},
                      {"key": "dc.title", "value":
                       "Monitoring Works: Getting Teachers",
                       "language": "en_US"},
                      {"key": "dc.relation.isversionof",
                       "value": "repo/0/ao/123"}]}]
     coll_id = '789'
     ingest_type = 'local'
-    file_dict = {'123': sample_content_1}
+    file_dict = {'test_01': f'{sample_files_dir}/test_01.pdf'}
     item_ids = client.post_items_to_coll(coll_id, coll_metadata, file_dict,
                                          ingest_type)
     for item_id in item_ids:
         assert 'a1b2' == item_id
 
 
-def test_post_bitstreams_to_item(client, sample_content_1, sample_content_2):
+def test_post_bitstreams_to_item(client, sample_files_dir):
     """Test post_bitstreams_to_item method."""
     item_id = 'a1b2'
     ingest_type = 'local'
     file_identifier = '123'
-    file_dict = {'123': sample_content_1}
+    file_dict = {'test_02': f'{sample_files_dir}/test_02.pdf',
+                 'test_01': f'{sample_files_dir}/test_01.pdf'}
     bit_ids = client.post_bitstreams_to_item(item_id, file_identifier,
                                              file_dict, ingest_type)
+    bit_ids_output = []
     for bit_id in bit_ids:
-        assert 'c3d4' == bit_id
+        bit_ids_output.append(bit_id)
+    assert bit_ids_output[0] == 'c3d4'
+    assert bit_ids_output[1] == 'e5f6'
+
+
+def test_post_bitstream(client, sample_files_dir):
+    """Test post_bitstream method."""
+    item_id = 'a1b2'
+    ingest_type = 'local'
+    file_identifier = '123'
+    file_dict = {'test_01': f'{sample_files_dir}/test_01.pdf'}
+    bitstream = 'test_01'
+    bit_id = client.post_bitstream(item_id, file_identifier, file_dict,
+                                   ingest_type, bitstream)
+    assert 'c3d4' == bit_id
 
 
 def test__pop_inst(client):

tests/test_workflows.py

@@ -0,0 +1,79 @@
+import csv
+
+from dsaps import workflows
+
+
+def test_create_file_dict_and_id_list(runner, sample_files_dir):
+    """Test create_file_dict_and_id_list function."""
+    file_path = sample_files_dir
+    file_dict, file_ids = workflows.create_file_dict_and_list(sample_files_dir,
+                                                              'pdf')
+    assert file_dict['test_02'] == f'{file_path}/test_02.pdf'
+    assert file_dict['test_01'] == f'{file_path}/test_01.pdf'
+    assert file_dict['best_01'] == f'{file_path}/best_01.pdf'
+    for id in ['test_02', 'test_01', 'best_01']:
+        assert id in file_ids
+
+
+def test_create_metadata_id_list(runner, sample_files_dir):
+    """Test create_metadata_id_list function."""
+    metadata_path = f'{sample_files_dir}/metadata.csv'
+    metadata_ids = workflows.create_metadata_id_list(metadata_path)
+    assert 'test' in metadata_ids
+
+
+def test_match_files_to_metadata():
+    """Test match_files_to_metadata function."""
+    file_dict = {'test_01': 'files/test_01.pdf'}
+    file_ids = ['test_01']
+    metadata_ids = ['test', 'tast']
+    file_matches = workflows.match_files_to_metadata(file_dict, file_ids,
+                                                     metadata_ids)
+    assert len(file_matches) == 1
+    assert 'test_01' in file_matches
+
+
+def test_match_metadata_to_files():
+    """Test match_metadata_to_files function."""
+    file_dict = {'test_01': 'files/test_01.pdf',
+                 'tast_01': 'files/tast_01.pdf'}
+    metadata_ids = ['test']
+    file_matches = workflows.match_metadata_to_files(file_dict, metadata_ids)
+    assert len(file_matches) == 1
+    assert 'test' in file_matches
+
+
+def test_reconcile_files_and_metadata(runner, sample_files_dir):
+    """Test reconcile function."""
+    with runner.isolated_filesystem():
+        metadata_path = f'{sample_files_dir}/metadata.csv'
+        workflows.reconcile_files_and_metadata(metadata_path, sample_files_dir,
+                                               'pdf')
+        with open('updated-metadata.csv') as csvfile2:
+            reader = csv.DictReader(csvfile2)
+            for row in reader:
+                assert row['uri'] == '/repo/0/ao/123'
+                assert row['title'] == 'Test Item'
+                assert row['file_identifier'] == 'test'
+        with open('no_metadata.csv') as csvfile3:
+            reader = csv.DictReader(csvfile3)
+            for row in reader:
+                assert row['id'] == 'best_01'
+        with open('no_files.csv') as csvfile4:
+            reader = csv.DictReader(csvfile4)
+            for row in reader:
+                assert row['id'] == 'tast'
+
+
+def test_update_metadata_csv(runner, sample_files_dir):
+    """Test update_metadata_csv function."""
+    with runner.isolated_filesystem():
+        metadata_matches = ['test']
+        workflows.update_metadata_csv(f'{sample_files_dir}/metadata.csv',
+                                      metadata_matches)
+        with open('updated-metadata.csv') as csvfile2:
+            reader = csv.DictReader(csvfile2)
+            for row in reader:
+                assert row['uri'] == '/repo/0/ao/123'
+                assert row['title'] == 'Test Item'
+                assert row['file_identifier'] == 'test'