changed item retrieval method & added skipped collections

ehanson8 · ehanson8 · commit c677d0b75765 · 2018-12-17T13:59:30.000-05:00
diff --git a/README.md b/README.md
@@ -10,29 +10,12 @@ All of these scripts require a secrets.py file in the same directory that must c
         filePath = '/Users/dspace_user/dspace-data-collection/data/'
         handlePrefix = 'http://dspace.myuni.edu/handle/'
         verify = True or False (no quotes). Use False if using an SSH tunnel to connect to the DSpace API
+        skippedCollections = A list of the 'uuid' of any collections that you wish the script to skip. (e.g. ['45794375-6640-4efe-848e-082e60bae375'])
 ```
 The 'filePath' is directory into which output files will be written and 'handlePrefix' may or may not vary from your DSpace URL depending on your configuration. This secrets.py file will be ignored according to the repository's .gitignore file so that DSpace login details will not be inadvertently exposed through GitHub.
 
 If you are using both a development server and a production server, you can create a separate secrets.py file with a different name (e.g. secretsProd.py) and containing the production server information. When running each of these scripts, you will be prompted to enter the file name (e.g 'secretsProd' without '.py') of an alternate secrets file. If you skip the prompt or incorrectly type the file name, the scripts will default to the information in the secrets.py file. This ensures that you will only access the production server if you really intend to.
 
-**Note**: All of these scripts skip collection '45794375-6640-4efe-848e-082e60bae375' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
-
-
-Skips collection 45794375-6640-4efe-848e-082e60bae375:
-
-                for j in range (0, len (collections)):
-                        collectionID = collections[j]['uuid']
-                        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
-                                offset = 0
-
-
-No collections skipped:
-
-                for j in range (0, len (collections)):
-                        collectionID = collections[j]['uuid']
-                        if collectionID != 0:
-                                offset = 0
-
 #### [compareTwoKeysInCommunity.py](compareTwoKeysInCommunity.py)
 Based on user input, extracts the values of two specified keys from a specified community to a CSV file for comparison.
 
diff --git a/compareTwoKeysInCommunity.py b/compareTwoKeysInCommunity.py
@@ -15,7 +15,7 @@
         print 'Editing Stage'
 else:
     print 'Editing Stage'
-    
+
 parser = argparse.ArgumentParser()
 parser.add_argument('-1', '--key', help='the first key to be output. optional - if not provided, the script will ask for input')
 parser.add_argument('-2', '--key2', help='the second key to be output. optional - if not provided, the script will ask for input')
@@ -43,6 +43,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -64,7 +65,7 @@
 for j in range (0, len (collections)):
     collectionID = collections[j]['uuid']
     print collectionID
-    if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+    if collectionID not in skippedCollections:
         offset = 0
         items = ''
         while items != []:
diff --git a/countInitialedNamesByCollection.py b/countInitialedNamesByCollection.py
@@ -23,6 +23,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -43,7 +44,7 @@
     collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for collection in collections:
         collectionID = collection['uuid']
-        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+        if collectionID not in skippedCollections:
             collectionIds.append(collectionID)
 
 names = []
diff --git a/exportCollectionMetadataToCSV.py b/exportCollectionMetadataToCSV.py
@@ -23,6 +23,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-i', '--handle', help='handle of the collection to retreive. optional - if not provided, the script will ask for input')
diff --git a/exportSelectedRecordMetadataToCSV.py b/exportSelectedRecordMetadataToCSV.py
@@ -23,6 +23,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-f', '--fileName', help='the CSV file of record handles. optional - if not provided, the script will ask for input')
diff --git a/findBogusUris.py b/findBogusUris.py
@@ -23,6 +23,7 @@
 filePath = secrets.filePath
 handlePrefix = secrets.handlePrefix
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
diff --git a/findDuplicateKeys.py b/findDuplicateKeys.py
@@ -32,6 +32,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 searchString = "\""+key+"\""
 
diff --git a/getBitstreams.py b/getBitstreams.py
@@ -98,6 +98,7 @@ def main():
 
     if not args.verify:
         args.verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
     if args.handle:
         handle = args.handle
diff --git a/getCollectionMetadataJson.py b/getCollectionMetadataJson.py
@@ -21,6 +21,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 handle = raw_input('Enter handle: ')
 
diff --git a/getCompleteAndUniqueValuesForAllKeys.py b/getCompleteAndUniqueValuesForAllKeys.py
@@ -25,6 +25,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 filePathComplete = filePath+'completeValueLists'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'/'
 filePathUnique = filePath+'uniqueValueLists'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'/'
@@ -48,7 +49,7 @@
     collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['uuid']
-        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+        if collectionID not in skippedCollections:
             collectionIds.append(collectionID)
 
 os.mkdir(filePathComplete)
diff --git a/getCompleteAndUniqueValuesForAllKeysInCommunity.py b/getCompleteAndUniqueValuesForAllKeysInCommunity.py
@@ -25,6 +25,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 handle = raw_input('Enter community handle: ')
 
@@ -51,7 +52,7 @@
 collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
 for j in range (0, len (collections)):
     collectionID = collections[j]['uuid']
-    if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+    if collectionID not in skippedCollections:
         offset = 0
         items = ''
         while items != []:
diff --git a/getFacultyNamesFromETDs.py b/getFacultyNamesFromETDs.py
@@ -33,6 +33,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
diff --git a/getGlobalLanguageValues.py b/getGlobalLanguageValues.py
@@ -20,6 +20,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -40,7 +41,7 @@
     collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['uuid']
-        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+        if collectionID not in skippedCollections:
             offset = 0
             items = ''
             while items != []:
diff --git a/getHandlesAndBitstreamsFromCollection.py b/getHandlesAndBitstreamsFromCollection.py
@@ -22,6 +22,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 handle = raw_input('Enter handle: ')
 
diff --git a/getLanguageValuesForKeys.py b/getLanguageValuesForKeys.py
@@ -22,6 +22,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -42,7 +43,7 @@
     collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['uuid']
-        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+        if collectionID not in skippedCollections:
             offset = 0
             items = ''
             while items != []:
diff --git a/getRecordsAndValuesForKey.py b/getRecordsAndValuesForKey.py
@@ -32,6 +32,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -49,6 +50,7 @@
 offset = 0
 recordsEdited = 0
 items = ''
+itemLinks = []
 while items != []:
     endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=exists&query_val[]=&limit=200&offset='+str(offset)
     print endpoint
@@ -57,17 +59,18 @@
     for item in items:
         itemMetadataProcessed = []
         itemLink = item['link']
-        metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
-        for l in range (0, len (metadata)):
-            if metadata[l]['key'] == key:
-                metadataValue = metadata[l]['value'].encode('utf-8')
-                for l in range (0, len (metadata)):
-                    if metadata[l]['key'] == 'dc.identifier.uri':
-                        uri = metadata[l]['value']
-                f.writerow([itemLink]+[uri]+[metadataValue])
+        itemLinks.append(itemLink)
     offset = offset + 200
     print offset
-
+for itemLink in itemLinks:
+    metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
+    for l in range (0, len (metadata)):
+        if metadata[l]['key'] == key:
+            metadataValue = metadata[l]['value'].encode('utf-8')
+            for l in range (0, len (metadata)):
+                if metadata[l]['key'] == 'dc.identifier.uri':
+                    uri = metadata[l]['value']
+            f.writerow([itemLink]+[uri]+[metadataValue])
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
 elapsedTime = time.time() - startTime
diff --git a/getRecordsAndValuesForKeyInCollection.py b/getRecordsAndValuesForKeyInCollection.py
@@ -37,6 +37,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -59,6 +60,7 @@
 offset = 0
 recordsEdited = 0
 items = ''
+itemLinks = []
 while items != []:
     endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=exists&query_val[]='+collSels+'&limit=200&offset='+str(offset)
     print endpoint
@@ -67,16 +69,18 @@
     for item in items:
         itemMetadataProcessed = []
         itemLink = item['link']
-        metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
-        for l in range (0, len (metadata)):
-            if metadata[l]['key'] == key:
-                metadataValue = metadata[l]['value'].encode('utf-8')
-                for l in range (0, len (metadata)):
-                    if metadata[l]['key'] == 'dc.identifier.uri':
-                        uri = metadata[l]['value']
-                f.writerow([itemLink]+[uri]+[metadataValue])
+        itemLinks.append(itemLink)
     offset = offset + 200
     print offset
+for itemLink in itemLinks:
+    metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
+    for l in range (0, len (metadata)):
+        if metadata[l]['key'] == key:
+            metadataValue = metadata[l]['value'].encode('utf-8')
+            for l in range (0, len (metadata)):
+                if metadata[l]['key'] == 'dc.identifier.uri':
+                    uri = metadata[l]['value']
+            f.writerow([itemLink]+[uri]+[metadataValue])
 
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
diff --git a/getRecordsWithKeyAndValue.py b/getRecordsWithKeyAndValue.py
@@ -37,6 +37,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -54,6 +55,7 @@
 offset = 0
 recordsEdited = 0
 items = ''
+itemLinks = []
 while items != []:
     endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=equals&query_val[]='+value+'&limit=200&offset='+str(offset)
     print endpoint
@@ -62,16 +64,18 @@
     for item in items:
         itemMetadataProcessed = []
         itemLink = item['link']
-        metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
-        for i in range (0, len (metadata)):
-            if metadata[i]['key'] == key and metadata[i]['value'] == value:
-                metadataValue = metadata[i]['value']
-                for i in range (0, len (metadata)):
-                    if metadata[i]['key'] == 'dc.identifier.uri':
-                        uri = metadata[i]['value']
-                f.writerow([itemLink]+[uri]+[key]+[metadataValue])
+        itemLinks.append(itemLink)
     offset = offset + 200
     print offset
+for itemLink in itemLinks:
+    metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
+    for i in range (0, len (metadata)):
+        if metadata[i]['key'] == key and metadata[i]['value'] == value:
+            metadataValue = metadata[i]['value']
+            for i in range (0, len (metadata)):
+                if metadata[i]['key'] == 'dc.identifier.uri':
+                    uri = metadata[i]['value']
+            f.writerow([itemLink]+[uri]+[key]+[metadataValue])
 
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
diff --git a/identifyItemsMissingKeyInCommunity.py b/identifyItemsMissingKeyInCommunity.py
@@ -39,6 +39,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 startTime = time.time()
 data = {'email':email,'password':password}
@@ -66,6 +67,7 @@
 offset = 0
 recordsEdited = 0
 items = ''
+itemLinks = []
 while items != []:
     endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=doesnt_exist&query_val[]='+collSels+'&limit=200&offset='+str(offset)
     print endpoint
@@ -74,13 +76,14 @@
     for item in items:
         itemMetadataProcessed = []
         itemLink = item['link']
-        metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
-        for metadataElement in metadata:
-            itemMetadataProcessed.append(metadataElement['key'])
-        if key not in itemMetadataProcessed:
-            f.writerow([itemLink])
     offset = offset + 200
     print offset
+for itemLink in itemLinks:
+    metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
+    for metadataElement in metadata:
+        itemMetadataProcessed.append(metadataElement['key'])
+    if key not in itemMetadataProcessed:
+        f.writerow([itemLink])
 
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
diff --git a/metadataCollectionsKeysMatrix.py b/metadataCollectionsKeysMatrix.py
@@ -24,6 +24,7 @@
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
+skippedCollections = secrets.skippedCollections
 
 #authentication
 startTime = time.time()
@@ -50,7 +51,7 @@
     for j in range (0, len (collections)):
         collectionID = collections[j]['uuid']
         print collectionID
-        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
+        if collectionID not in skippedCollections:
             offset = 0
             items = ''
             while items != []:
diff --git a/metadataOverview.py b/metadataOverview.py
