updates

Author: ehanson8

getRecordsWithKeyAndValue.py

@@ -4,6 +4,21 @@
 import csv
 import time
 import urllib3
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-k', '--key', help='the key to be searched. optional - if not provided, the script will ask for input')
+parser.add_argument('-v', '--value', help='the value to be searched. optional - if not provided, the script will ask for input')
+args = parser.parse_args()
+
+if args.key:
+    key = args.key
+else:
+    key = raw_input('Enter the key: ')
+if args.value:
+    value = args.value
+else:
+    value = raw_input('Enter the value: ')
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -23,9 +38,6 @@
 filePath = secrets.filePath
 verify = secrets.verify
 
-key = raw_input('Enter key: ')
-value = raw_input('Enter value: ')
-
 startTime = time.time()
 data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
@@ -37,49 +49,29 @@
 userFullName = status['fullname']
 print 'authenticated'
 
-endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
-
-itemList = []
-endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
-for i in range (0, len (communities)):
-    communityID = communities[i]['uuid']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
-    for j in range (0, len (collections)):
-        collectionID = collections[j]['uuid']
-        if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
-            offset = 0
-            items = ''
-            while items != []:
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
-                while items.status_code != 200:
-                    time.sleep(5)
-                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
-                items = items.json()
-                for k in range (0, len (items)):
-                    itemID = items[k]['uuid']
-                    itemList.append(itemID)
-                offset = offset + 200
-elapsedTime = time.time() - startTime
-m, s = divmod(elapsedTime, 60)
-h, m = divmod(m, 60)
-print 'Item list creation time: ','%d:%02d:%02d' % (h, m, s)
-
 f=csv.writer(open(filePath+'Key='+key+' Value='+value+'.csv', 'wb'))
 f.writerow(['itemID']+['uri']+['key']+['value'])
-for number, itemID in enumerate(itemList):
-    itemMetadataProcessed = []
-    itemsRemaining = len(itemList) - number
-    print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
-    for i in range (0, len (metadata)):
-        if metadata[i]['key'] == key and metadata[i]['value'] == value:
-            metadataValue = metadata[i]['value']
-            for i in range (0, len (metadata)):
-                if metadata[i]['key'] == 'dc.identifier.uri':
-                    uri = metadata[i]['value']
-            f.writerow([itemID]+[uri]+[key]+[metadataValue])
+offset = 0
+recordsEdited = 0
+items = ''
+while items != []:
+    endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=equals&query_val[]='+value+'&limit=200&offset='+str(offset)
+    print endpoint
+    response = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
+    items = response['items']
+    for item in items:
+        itemMetadataProcessed = []
+        itemLink = item['link']
+        metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
+        for i in range (0, len (metadata)):
+            if metadata[i]['key'] == key and metadata[i]['value'] == value:
+                metadataValue = metadata[i]['value']
+                for i in range (0, len (metadata)):
+                    if metadata[i]['key'] == 'dc.identifier.uri':
+                        uri = metadata[i]['value']
+                f.writerow([itemLink]+[uri]+[key]+[metadataValue])
+    offset = offset + 200
+    print offset
 
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
 

identifyDuplicateKeyValuePairsFromItemsDiffLangTags.py

@@ -43,7 +43,7 @@
     collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['uuid']
-        if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
+        if collectionID != '45794375-6640-4efe-848e-082e60bae375':
             offset = 0
             items = ''
             while items != []:
@@ -61,7 +61,7 @@
 h, m = divmod(m, 60)
 print 'Item list creation time: ','%d:%02d:%02d' % (h, m, s)
 
-f=csv.writer(open(filePath+'DuplicatesRecordsDiffLangTags'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
+f=csv.writer(open(filePath+'DuplicateKeysWithDiffLangTags'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
 f.writerow(['itemID']+['key:value'])
 for number, itemID in enumerate(itemList):
     itemMetadataProcessed = []

identifyItemWithDuplicateKeysInCommunity.py

@@ -5,6 +5,22 @@
 import csv
 from datetime import datetime
 import urllib3
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-k', '--key', help='the key to be searched. optional - if not provided, the script will ask for input')
+parser.add_argument('-i', '--handle', help='handle of the community to retreive. optional - if not provided, the script will ask for input')
+args = parser.parse_args()
+
+if args.key:
+    key = args.key
+else:
+    key = raw_input('Enter the key to be searched: ')
+
+if args.handle:
+    handle = args.handle
+else:
+    handle = raw_input('Enter collection handle: ')
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -24,9 +40,6 @@
 filePath = secrets.filePath
 verify = secrets.verify
 
-handle = raw_input('Enter community handle: ')
-key = raw_input('Enter key: ')
-
 startTime = time.time()
 data = {'email':email,'password':password}
 header = {'content-type':'application/json','accept':'application/json'}
@@ -38,46 +51,36 @@
 userFullName = status['fullname']
 print 'authenticated'
 
-itemList = []
 endpoint = baseURL+'/rest/handle/'+handle
 community = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
 communityID = community['uuid']
-
 collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
+collSels = ''
 for j in range (0, len (collections)):
     collectionID = collections[j]['uuid']
-    if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
-        offset = 0
-        items = ''
-        while items != []:
-            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
-            while items.status_code != 200:
-                time.sleep(5)
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
-            items = items.json()
-            for k in range (0, len (items)):
-                itemID = items[k]['uuid']
-                itemList.append(itemID)
-            offset = offset + 200
-elapsedTime = time.time() - startTime
-m, s = divmod(elapsedTime, 60)
-h, m = divmod(m, 60)
-print 'Item list creation time: ','%d:%02d:%02d' % (h, m, s)
+    collSel = '&collSel[]=' + collectionID
+    collSels = collSels + collSel
 
 f=csv.writer(open(filePath+'recordsMissing'+key+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
 f.writerow(['itemID']+['key'])
-idList =[]
-for number, itemID in enumerate(itemList):
-    itemMetadataProcessed = []
-    itemsRemaining = len(itemList) - number
-    print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
-    for metadataElement in metadata:
-        itemMetadataProcessed.append(metadataElement['key'])
-    if key not in itemMetadataProcessed:
-        f.writerow([itemID])
-        idList.append(itemID)
-print idList
+offset = 0
+recordsEdited = 0
+items = ''
+while items != []:
+    endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=doesnt_exist&query_val[]='+collSels+'&limit=200&offset='+str(offset)
+    print endpoint
+    response = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
+    items = response['items']
+    for item in items:
+        itemMetadataProcessed = []
+        itemLink = item['link']
+        metadata = requests.get(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify).json()
+        for metadataElement in metadata:
+            itemMetadataProcessed.append(metadataElement['key'])
+        if key not in itemMetadataProcessed:
+            f.writerow([itemLink])
+    offset = offset + 200
+    print offset
 
 logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)