updates

Author: ehanson8

README.md

@@ -1,5 +1,7 @@
 # dspace-data-collection
 
+**Note**: These scripts were updated in 05/2018 for the new authentication method used by DSpace 6.x
+
 All of these scripts require a secrets.py file in the same directory that must contain the following text:
 ```
         baseURL='https://dspace.myuni.edu'
@@ -13,23 +15,21 @@ The 'filePath' is directory into which output files will be written and 'handleP
 
 If you are using both a development server and a production server, you can create a separate secrets.py file with a different name (e.g. secretsProd.py) and containing the production server information. When running each of these scripts, you will be prompted to enter the file name (e.g 'secretsProd' without '.py') of an alternate secrets file. If you skip the prompt or incorrectly type the file name, the scripts will default to the information in the secrets.py file. This ensures that you will only access the production server if you really intend to.
 
-The command 'requests.packages.urllib3.disable_warnings()' is used to disable the excessive warnings that will be produced if the 'verify' variable is set to False, which necessary if you are using an SSH tunnel to connect to the DSpace API.
-
-**Note**: All of these scripts skip collection '24' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
+**Note**: All of these scripts skip collection '4dccec82-4cfb-4583-a728-2cb823b15ef0' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
 
 
-Skips collection 24:
+Skips collection 4dccec82-4cfb-4583-a728-2cb823b15ef0:
 
                 for j in range (0, len (collections)):
-                        collectionID = collections[j]['id']
-                        if collectionID != 24:
+                        collectionID = collections[j]['uuid']
+                        if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
                         offset = 0
 
 
 No collections skipped:
 
                 for j in range (0, len (collections)):
-                        collectionID = collections[j]['id']
+                        collectionID = collections[j]['uuid']
                         if collectionID != 0:
                         offset = 0
 

compareTwoKeysInCommunity.py

@@ -3,6 +3,9 @@
 import secrets
 import csv
 import time
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
 if secretsVersion != '':
@@ -13,44 +16,50 @@
         print 'Editing Stage'
 else:
     print 'Editing Stage'
-    
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
 
-requests.packages.urllib3.disable_warnings()
-
-communityID = raw_input('Enter community ID: ')
+communityHandle = raw_input('Enter community handle: ')
 key = raw_input('Enter first key: ')
 key2 = raw_input('Enter second key: ')
 
 startTime = time.time()
-data = json.dumps({'email':email,'password':password})
+data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
-headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
+cookies = {'JSESSIONID': session}
+headerFileUpload = {'accept':'application/json'}
+cookiesFileUpload = cookies
+status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
 print 'authenticated'
 
+endpoint = baseURL+'/rest/handle/'+communityHandle
+community = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
+communityID = community['uuid']
+
 itemList = []
 endpoint = baseURL+'/rest/communities'
-collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
+collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
 for j in range (0, len (collections)):
-    collectionID = collections[j]['id']
-    if collectionID != 24:
+    collectionID = collections[j]['uuid']
+    print collectionID
+    if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
         offset = 0
         items = ''
         while items != []:
-            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
+            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
             while items.status_code != 200:
                 time.sleep(5)
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
             items = items.json()
             for k in range (0, len (items)):
-                itemID = items[k]['id']
+                itemID = items[k]['uuid']
                 itemList.append(itemID)
-            offset = offset + 1000
+            offset = offset + 200
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)
 h, m = divmod(m, 60)
@@ -60,16 +69,16 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
     itemTuple = (itemID,)
     tupleValue1 = ''
     tupleValue2 = ''
     for l in range (0, len (metadata)):
         if metadata[l]['key'] == key:
-            metadataValue = metadata[l]['value']
+            metadataValue = metadata[l]['value'].encode('utf-8')
             tupleValue1 = metadataValue
         if metadata[l]['key'] == key2:
-            metadataValue = metadata[l]['value']
+            metadataValue = metadata[l]['value'].encode('utf-8')
             tupleValue2 = metadataValue
     itemTuple = itemTuple + (tupleValue1 , tupleValue2)
     valueList.append(itemTuple)
@@ -86,7 +95,7 @@
 for i in range (0, len (valueList)):
     f.writerow([valueList[i][0]]+[valueList[i][1]]+[valueList[i][2]])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
+logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

findBogusUris.py

@@ -3,6 +3,9 @@
 import secrets
 import csv
 import time
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
 if secretsVersion != '':
@@ -13,44 +16,48 @@
         print 'Editing Stage'
 else:
     print 'Editing Stage'
-    
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
 handlePrefix = secrets.handlePrefix
 verify = secrets.verify
 
-requests.packages.urllib3.disable_warnings()
-
 startTime = time.time()
-data = json.dumps({'email':email,'password':password})
+data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
-headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
+cookies = {'JSESSIONID': session}
+headerFileUpload = {'accept':'application/json'}
+cookiesFileUpload = cookies
+status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
+userFullName = status['fullname']
 print 'authenticated'
 
 itemList = []
 endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
+communities = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
 for i in range (0, len (communities)):
-    communityID = communities[i]['id']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
+    communityID = communities[i]['uuid']
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
-        collectionID = collections[j]['id']
-        if collectionID != 24:
+        collectionID = collections[j]['uuid']
+        print collectionID
+        if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
             offset = 0
             items = ''
             while items != []:
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
                 while items.status_code != 200:
                     time.sleep(5)
-                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
+                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
                 items = items.json()
                 for k in range (0, len (items)):
-                    itemID = items[k]['id']
+                    itemID = items[k]['uuid']
                     itemList.append(itemID)
-                offset = offset + 1000
+                offset = offset + 200
+                print offset
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)
 h, m = divmod(m, 60)
@@ -62,14 +69,14 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
     for l in range (0, len (metadata)):
         if metadata[l]['key'] == 'dc.identifier.uri':
             uri = str(metadata[l]['value'])
             if uri.startswith(handlePrefix) == False:
                 f.writerow([itemID]+[uri])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
+logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

findDuplicateKeys.py

@@ -3,6 +3,9 @@
 import secrets
 import time
 import csv
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
 if secretsVersion != '':
@@ -20,36 +23,43 @@
 filePath = secrets.filePath
 verify = secrets.verify
 
-requests.packages.urllib3.disable_warnings()
-
 key = raw_input('Enter key: ')
 searchString = "\""+key+"\""
 
 startTime = time.time()
-data = json.dumps({'email':email,'password':password})
+data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
-headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
+cookies = {'JSESSIONID': session}
+headerFileUpload = {'accept':'application/json'}
+cookiesFileUpload = cookies
+status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
+userFullName = status['fullname']
 print 'authenticated'
 
-
 itemList = []
 endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
+communities = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
 for i in range (0, len (communities)):
-    communityID = communities[i]['id']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
+    communityID = communities[i]['uuid']
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
-        collectionID = collections[j]['id']
-        if collectionID != 24:
-            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth, verify=verify)
-            while items.status_code != 200:
-                time.sleep(5)
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth, verify=verify)
-            items = items.json()
-            for k in range (0, len (items)):
-                itemID = items[k]['id']
-                itemList.append(itemID)
+        collectionID = collections[j]['uuid']
+        print collectionID
+        if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
+            offset = 0
+            items = ''
+            while items != []:
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
+                while items.status_code != 200:
+                    time.sleep(5)
+                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
+                items = items.json()
+                for k in range (0, len (items)):
+                    itemID = items[k]['uuid']
+                    itemList.append(itemID)
+                offset = offset + 200
+                print offset
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)
 h, m = divmod(m, 60)
@@ -60,12 +70,12 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
     metadata = json.dumps(metadata)
     if metadata.find(searchString) != metadata.rfind(searchString):
         f.writerow([itemID])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
+logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

getCollectionMetadataJson.py

@@ -2,6 +2,9 @@
 import requests
 import secrets
 import time
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
 if secretsVersion != '':
@@ -12,45 +15,47 @@
         print 'Editing Stage'
 else:
     print 'Editing Stage'
-    
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
 verify = secrets.verify
 
-requests.packages.urllib3.disable_warnings()
-
 handle = raw_input('Enter handle: ')
 
-data = json.dumps({'email':email,'password':password})
+startTime = time.time()
+data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
-headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
+cookies = {'JSESSIONID': session}
+headerFileUpload = {'accept':'application/json'}
+cookiesFileUpload = cookies
+status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
+userFullName = status['fullname']
 print 'authenticated'
-startTime = time.time()
 
 endpoint = baseURL+'/rest/handle/'+handle
-collection = requests.get(endpoint, headers=headerAuth, verify=verify).json()
-collectionID = collection['id']
-collectionTitle = requests.get(endpoint, headers=headerAuth, verify=verify).json()
+collection = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
+collectionID = collection['uuid']
+collectionTitle = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
 endpoint = baseURL+'/rest/collections/'+str(collectionID)+'/items'
-output = requests.get(endpoint, headers=headerAuth, verify=verify).json()
+output = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
 
 itemList = []
 for i in range (0, len (output)):
     name = output[i]['name']
-    itemID = output[i]['id']
+    itemID = output[i]['uuid']
     itemList.append(itemID)
 
 f=open(filePath+handle.replace('/','-')+'.json', 'w')
 metadataGroup = []
 for itemID in itemList:
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
     metadataGroup.append(metadata)
 json.dump(metadataGroup, f)
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
+logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)