Merge pull request #15 from QuantGov/add-series

Version 0.2.1

Author: jnelson16

regcensus/api.py

@@ -102,7 +102,7 @@ def get_values(series, jurisdiction, date, filtered=True, summary=True,
     # and function returns empty.
     else:
         print("Valid date is required.")
-        pp.pprint(get_periods(jurisdiction))
+        pp.pprint(get_periods(jurisdiction, documentType=3))
         return
 
     if dateIsRange:
@@ -151,82 +151,88 @@ def get_values(series, jurisdiction, date, filtered=True, summary=True,
         return clean_columns(output)
 
 
-def get_series(seriesID=''):
+def get_series(seriesID='', verbose=0):
     """
     Get metadata for all or one specific series
 
     Args: seriesID (optional): ID for the series
 
     Returns: pandas dataframe with the metadata
     """
-    output = json_normalize(
-        requests.get(URL + f'/series/{seriesID}').json())
-    return clean_columns(output)
+    url_call = URL + f'/series/{seriesID}'
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
-def get_agencies(jurisdictionID):
+def get_agencies(jurisdictionID, verbose=0):
     """
     Get metadata for all agencies of a specific jurisdiction
 
     Args: jurisdictionID: ID for the jurisdiction
 
     Returns: pandas dataframe with the metadata
     """
-    output = json_normalize(
-        requests.get(
-            URL + (f'/agencies/jurisdiction?'
-                   f'jurisdictions={jurisdictionID}')).json())
-    return clean_columns(output)
+    url_call = URL + (f'/agencies/jurisdiction?'
+                      f'jurisdictions={jurisdictionID}')
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
-def get_jurisdictions(jurisdictionID=''):
+def get_jurisdictions(jurisdictionID='', verbose=0):
     """
     Get metadata for all or one specific jurisdiction
 
     Args: jurisdictionID (optional): ID for the jurisdiction
 
     Returns: pandas dataframe with the metadata
     """
-    output = json_normalize(
-        requests.get(URL + f'/jurisdictions/{jurisdictionID}').json())
-    return clean_columns(output)
+    url_call = URL + f'/jurisdictions/{jurisdictionID}'
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
-def get_periods(jurisdictionID='', documentType=3):
+def get_periods(jurisdictionID, documentType, seriesID='', verbose=0):
     """
-    Get dates available for all or one specific jurisdiction
-    and compatible series IDs
+    Get dates available for all or one specific series for a jurisdiction
 
-    Args: jurisdictionID (optional): ID for the jurisdiction
+    Args:
+        jurisdictionID: ID for the jurisdiction
+        documentType: document type (regulations, statutes, etc.),
+            see list_document_types()
+        seriesID (optional): ID for the series
 
     Returns: pandas dataframe with the dates
     """
-    if jurisdictionID:
-        output = json_normalize(
-            requests.get(
-                URL + (f'/periods?jurisdiction={jurisdictionID}&'
-                       f'documentType={documentType}')).json())
+    if seriesID:
+        url_call = (URL + (f'/periods?jurisdiction={jurisdictionID}&'
+                           f'series={seriesID}&'
+                           f'documentType={documentType}'))
     else:
-        output = json_normalize(
-            requests.get(URL + f'/periods/available').json())
-    return clean_columns(output)
+        url_call = (URL + (f'/periods?jurisdiction={jurisdictionID}&'
+                           f'documentType={documentType}'))
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
-def get_industries(jurisdictionID):
+def get_industries(jurisdictionID, verbose=0):
     """
     Get metadata for all industries available in a specific jurisdiction
 
     Args: jurisdictionID: ID for the jurisdiction
 
     Returns: pandas dataframe with the metadata
     """
-    output = json_normalize(
-        requests.get(
-            URL + f'/industries?jurisdiction={jurisdictionID}').json())
-    return clean_columns(output)
+    url_call = URL + f'/industries?jurisdiction={jurisdictionID}'
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
-def get_documents(jurisdictionID, documentType=3):
+def get_documents(jurisdictionID, documentType=3, verbose=0):
     """
     Get metadata for documents available in a specific jurisdiction, optional
     filtering by document type (see list_document_types() for options)
@@ -237,12 +243,11 @@ def get_documents(jurisdictionID, documentType=3):
 
     Returns: pandas dataframe with the metadata
     """
-    output = json_normalize(
-        requests.get(
-            URL + (f'/documents?jurisdiction={jurisdictionID}&'
-                   f'documentType={documentType}')
-        ).json())
-    return clean_columns(output)
+    url_call = URL + (f'/documents?jurisdiction={jurisdictionID}&'
+                      f'documentType={documentType}')
+    if verbose:
+        print(f'API call: {url_call}')
+    return clean_columns(json_normalize(requests.get(url_call).json()))
 
 
 def list_document_types():

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name='regcensus',
-    version='0.2.0',
+    version='0.2.1',
     description='Python package for accessing data from the QuantGov API',
     url='https://github.com/QuantGov/regcensus-api-python',
     author='QuantGov',

tests/test_api.py

@@ -4,53 +4,60 @@
 
 
 # UTILITY FUNCTIONS
-def order_results(results, column):
-    return list(results[column].sort_values().head(10).values)
+def order_results(results, column, descending=False):
+    if descending:
+        return list(reversed(results[column].sort_values().tail(10).values))
+    else:
+        return list(results[column].sort_values().head(10).values)
 
 
 # TEST FUNCTIONS
 def test_get_series():
-    results = rc.get_series()
+    results = rc.get_series(verbose=1)
     assert order_results(results, 'seriesCode') == [
         'NY.GDP.MKTP.CD', 'NY.GDP.MKTP.KD', 'NY.GDP.MKTP.KD.ZG',
-        'NY.GDP.PCAP.KD.ZG', 'RG_OCL1000002A', 'RG_OCLI1000001A',
-        'RG_QLTY1000470Z', 'RG_QLTY1000471Z',
-        'RG_QLTY1000472Z', 'RG_QLTY1000473Z'
+        'NY.GDP.PCAP.KD.ZG', 'RG_DRST0000001A', 'RG_HLC1000002A',
+        'RG_OCL1000002A', 'RG_OCL1000002B', 'RG_OCL1000002C', 'RG_OCLI1000001A'
     ]
 
 
 def test_get_agencies():
-    results = rc.get_agencies(38)
+    results = rc.get_agencies(38, verbose=1)
     assert order_results(results, 'agencyID') == [
-        64, 65, 66, 67, 68, 69, 70, 71, 72, 73
+        64, 65, 66, 67, 68, 70, 71, 72, 73, 74
     ]
 
 
 def test_get_jurisdictions():
-    results = rc.get_jurisdictions()
+    results = rc.get_jurisdictions(verbose=1)
     assert order_results(results, 'jurisdictionID') == [
         2, 4, 10, 11, 14, 15, 17, 20, 23, 24
     ]
 
 
 def test_get_periods():
-    results = rc.get_periods()
-    assert order_results(results, 'seresYearID') == [
-        102994, 102995, 102996, 102997, 102998,
-        102999, 103000, 103001, 103002, 103003
+    results = rc.get_periods(38, documentType=3, verbose=1)
+    assert order_results(results, 'recordsAvailable', descending=True) == [
+        30696278, 30696278, 30696278, 30696278, 30696278,
+        30696278, 30696278, 30696278, 30696278, 30696278
     ]
 
 
+def test_get_periods_one_series():
+    results = rc.get_periods(20, documentType=3, seriesID=19, verbose=1)
+    assert order_results(results, 'recordsAvailable') == [13]
+
+
 def test_get_industries():
-    results = rc.get_industries(jurisdictionID=38)
+    results = rc.get_industries(jurisdictionID=38, verbose=1)
     assert order_results(results, 'industryCode') == [
         '0', '11', '111', '1111', '11111', '111110',
         '11112', '111120', '11113', '111130'
     ]
 
 
 def test_get_documents():
-    results = rc.get_documents(jurisdictionID=44)
+    results = rc.get_documents(jurisdictionID=44, verbose=1)
     assert order_results(results, 'documentID') == [
         4441363, 4441364, 4441365, 4441366, 4441367,
         4441368, 4441369, 4441370, 4441371, 4441372
@@ -82,10 +89,10 @@ def test_get_values_all_industries():
         series=9, jurisdiction=58, date=2019, industry='all', filtered=False
     )
     assert order_results(results, 'seriesValue') == [
-        16.487800191811402, 28.080800290597836, 36.27130037093593,
-        36.53810011051246, 40.113500030507566, 45.02970027324045,
-        48.842899827621295, 50.17920009633963, 72.05880061667267,
-        82.19629916545819
+        16.487800191811402, 28.080800290597836, 32.408500283963804,
+        36.27130037093593, 36.53810011051246, 36.72170047096006,
+        40.113500030507566, 44.19190023323608, 45.02970027324045,
+        48.842899827621295
     ]
 
 
@@ -100,11 +107,14 @@ def test_get_values_multiple_industries():
 
 def test_get_values_one_industry():
     results = rc.get_values(
-        series=9, jurisdiction=58, date='2019-05-15',
+        series=9, jurisdiction=20, date='2020-06-02',
         industry='111', summary=False
     )
-    assert order_results(results, 'seriesValue') == [
-        0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
+    assert order_results(results, 'seriesValue', descending=True) == [
+        208.5117055773735, 177.00449323654175, 86.01110327243805,
+        76.680001989007, 66.70080256462097, 61.46759942173958,
+        58.94939732551575, 41.14539943635464, 36.53759956359863,
+        33.51419833302498
     ]
 
 
@@ -140,7 +150,7 @@ def test_get_values_incorrect_dates(capsys):
 def test_get_values_country():
     results = rc.get_values(series=1, jurisdiction=38, date=2019, country=True)
     assert order_results(results, 'seriesValue') == [
-        43940.0, 52569.0, 60086.0, 63203.0, 63735.0,
+        43940.0, 51925.0, 52569.0, 60086.0, 63735.0,
         70969.0, 78676.0, 92522.0, 104562.0, 107063.0
     ]
 
@@ -187,21 +197,26 @@ def test_get_values_error(capsys):
     results = rc.get_values(series=1, jurisdiction=38, date=1900)
     assert not results
     assert capsys.readouterr().out == (
-        'WARNING: SeriesValue was not found for the specified parameters'
-        '{parameters={jurisdiction=[38], date=[1900], industry=null, '
-        'agency=null, dateIsRange=false, filteredOnly=true, summary=true, '
-        'documentType=3, documentID=null}}\n'
+        'WARNING: SeriesValue was not found for the specified parameters. '
+        'Please check that you have selected the right combination of '
+        'parameters.  When in doubt, please use the /periods endpoint to '
+        'find out the combinations of series, jurisdiction, periods, '
+        'agencies, document types for which there are data available.'
+        '{parameters={jurisdiction=[US_UNITED_STATES], date=[1900], '
+        'industry=null, agency=null, dateIsRange=false, filteredOnly=true, '
+        'summary=true, series=[SERIES_1], documentType=ALL_REGULATIONS, '
+        'documentID=null}}\n'
     )
 
 
 def test_list_document_types():
     results = rc.list_document_types()
-    assert results['All Regulations'] == 3
+    assert results['Regulation text: All regulations'] == 3
 
 
 def test_list_series():
     results = rc.list_series()
-    assert results['Conditionals'] == 135
+    assert results['Complexity: Conditionals analysis'] == 135
 
 
 def test_list_agencies():