Merge pull request #94 from z3c0/test (v0.9.59)

v0.9.59

Author: z3c0

README.md

@@ -148,11 +148,11 @@ Currently, the only public datasets supported by Vistos are the [Biographical Di
 
 The `CongressMember` class exists for querying data from the perspective of members. `CongressMember` is a much faster option for when you know the specific member(s) you would like to download data for.
 
-`CongressMember` takes a Bioguide ID as an argument, and attempts to retrieve the specified data for the member associated with the given ID.
+`CongressMember` takes a Bioguide ID as an argument, and attempts to retrieve data for the member associated with the given ID.
 
 #### `.load()` <a name="member_load"></a>
 
-Manually load datasets specified when instantiating `CongressMember`
+The `load` method manually load the datasets specified when instantiating `CongressMember`
 
 ``` python
 member = v.CongressMember('P000587', load_immediately=False)
@@ -179,7 +179,7 @@ print(member.govinfo['title'])
 Senator Edward M. Kennedy, Biography
 ```
 
-Due to limitations in the GovInfo API, directly retrieving data about a member is not possible. Vistos attempts to work around these limitations by first requesting the members Bioguide data, and using the information found there to narrow down where to locate the member's data within the GovInfo API.
+Due to limitations in the GovInfo API, directly retrieving data about a member is not possible. Vistos attempts to work around these limitations by first requesting the members Bioguide data, and using the information found there to narrow down where to locate the member's data within the Congressional Directories of the Congresses the member belonged to.
 
 #### `.bioguide_id` <a name="member_bioguide_id"></a>
 
@@ -444,9 +444,11 @@ def main():
             print(f'[{bioguide_id}] Downloading...', end='\r')
             try:
                 member.load()
-                print(f'[{bioguide_id}] Downloaded    ')
+                print(f'[{bioguide_id}] Downloaded    ', end='')
             except v.gpo.BioguideConnectionError:
-                print(f'[{bioguide_id}] Download Failed')
+                print(f'[{bioguide_id}] Download Failed', end='')
+            finally:
+                print()
 
             # Split terms from the rest of the data
             member_header_record = dict(member.bioguide)
@@ -510,7 +512,7 @@ If you'd like to contribute to the project, or know of a useful data source, fee
 
 ## Known Issues and Workarounds <a name="issues"></a>
 
-1. Querying GovInfo not returning results for some congress persons.
+1. Querying GovInfo does not return results for some congress persons.
 
     When using the `CongressMember` class, some congress persons will not return GovInfo data. This is due to their GovInfo data missing a Bioguide ID, which is used for finding individual records within a Congressional Directory package. Should you need data for one of these members, the only workaround (currently) is to instead query GovInfo for the entire Congress that they served for, using the `Congress` class.
 

tests/test_integration.py

@@ -187,6 +187,12 @@ def test_bioguide_query_object(self):
         self.assertEqual(query.last_name, 'wren')
         self.assertEqual(query.first_name, 'thomas')
 
+    def test_congress_bills(self):
+        congress = v.Congress(105, self.GOVINFO_API_KEY)
+        congress.load_bills()
+        self.assertIsNotNone(congress.bills)
+        self.assertEqual(len(congress.bills), 13126)
+
 
 if __name__ == '__main__':
     VistosIntegrationTests.GOVINFO_API_KEY = config('GOVINFO_API_KEY')

vistos/VERSION

@@ -1 +1 @@
-0.9.56
+0.9.60

vistos/__init__.py

@@ -2,10 +2,10 @@
 """vistos (V)"""
 from os import path as _path
 
-from vistos.src import (Congress, CongressMember, search_bioguide_members,
-                        search_govinfo_members, gpo)
+from vistos.src import (Congress, CongressMember, CongressBills,
+                        search_bioguide_members, search_govinfo_members, gpo)
 
-__all__ = ['gpo', 'Congress', 'CongressMember', 'search_bioguide_members',
-           'search_govinfo_members']
+__all__ = ['gpo', 'Congress', 'CongressMember', 'CongressBills',
+           'search_bioguide_members', 'search_govinfo_members']
 
 VERSION = open(_path.join(_path.dirname(__file__), 'VERSION'), 'r').read()

vistos/src/__init__.py

@@ -1,6 +1,6 @@
 """Backbone for V module"""
-from vistos.src.duo import (Congress, CongressMember, search_bioguide_members,
-                            search_govinfo_members)
+from vistos.src.duo import (Congress, CongressMember, CongressBills,
+                            search_bioguide_members, search_govinfo_members)
 
-__all__ = ['Congress', 'CongressMember', 'search_bioguide_members',
-           'search_govinfo_members']
+__all__ = ['Congress', 'CongressMember', 'CongressBills',
+           'search_bioguide_members', 'search_govinfo_members']

vistos/src/duo.py

@@ -39,6 +39,27 @@ def search_govinfo_members(govinfo_api_key, first_name=None, last_name=None,
     return members_list
 
 
+class CongressBills(list):
+    """An object for query bills data for a single Congress"""
+
+    def __init__(self, congress_number, govinfo_api_key, bill_type=None,
+                 load_immediately=True):
+        self._bills = None
+
+        self._number = gpo.convert_to_congress_number(congress_number)
+        self._years = gpo.get_congress_years(self._number)
+
+        self._load_bills = \
+            gpo.govinfo.create_bills_func(govinfo_api_key, self._number)
+
+        if load_immediately:
+            self.load()
+
+    def load(self):
+        for bill in self._load_bills():
+            self.append(bill)
+
+
 class CongressMember:
     """An object for querying data for a single Congress member"""
 
@@ -245,6 +266,8 @@ def __init__(self, number_or_year=None, govinfo_api_key=None,
 
             if govinfo_exists:
                 self._enable_govinfo(govinfo_api_key)
+                self._bills = \
+                    CongressBills(self._number, govinfo_api_key, None, False)
             else:
                 include_bioguide = True
 
@@ -267,8 +290,8 @@ def load(self):
 
     def load_bills(self):
         """Manually load bills issued during the current Congress"""
-        if self.load_bills is not None:
-            self._bills = self._load_bills()
+        if self._bills is not None:
+            self._bills.load()
 
     def _enable_govinfo(self, key):
         """Enable loading govinfo when load() is called"""
@@ -335,11 +358,13 @@ def bioguide(self):
 
     @property
     def govinfo(self):
-        """returns GovInfo data as `GovInfoCongressRecord`"""
+        """returns GovInfo Congressional Directory data as a
+        `GovInfoCongressRecord`"""
         return self._gi
 
     @property
     def bills(self):
+        """Return GovInfo Bills data"""
         return self._bills
 
     @bioguide.setter

vistos/src/gpo/bioguideretro.py

@@ -88,7 +88,7 @@ def __init__(self, xml_data) -> None:
             str(xml_data.find('term-state').text).upper()
 
         party = xml_data.find('term-party').text
-        if party == 'NA' or party.strip() == '':
+        if party == 'NA' or (party and party.strip() == ''):
             self[_fields.Term.PARTY] = None
         else:
             self[_fields.Term.PARTY] = str(party).lower()

vistos/src/gpo/fields.py

@@ -36,6 +36,7 @@ class Term:
 
 class Bill:
     """Headers for GovInfoBillRecord"""
+    BILL_ID = 'bill_id'
     CONGRESS = 'congress_number'
     TITLE = 'title'
     SHORT_TITLE = 'short_title'

vistos/src/gpo/govinfo.py

@@ -19,24 +19,105 @@ class GovInfoBillRecord(dict):
 
     def __init__(self, bill_govinfo: dict):
         super().__init__()
+        self[_fields.Bill.BILL_ID] = (bill_govinfo['congress'] + '-' +
+                                      bill_govinfo['session'] + '-' +
+                                      bill_govinfo['billType'] + '-' +
+                                      bill_govinfo['billNumber'] + '-' +
+                                      bill_govinfo['billVersion'])
+
         self[_fields.Bill.TITLE] = bill_govinfo['title']
         try:
             self[_fields.Bill.SHORT_TITLE] = \
                 bill_govinfo['shortTitle'][0]['title']
         except (KeyError, IndexError):
             self[_fields.Bill.SHORT_TITLE] = None
 
+        self[_fields.Bill.CONGRESS] = bill_govinfo['congress']
+
         self[_fields.Bill.DATE_ISSUED] = bill_govinfo['dateIssued']
         self[_fields.Bill.PAGES] = bill_govinfo['pages']
-        self[_fields.Bill.GOVERNMENT_AUTHOR] = \
-            bill_govinfo['governmentAuthor2']
+        self[_fields.Bill.SESSION] = int(bill_govinfo['session'])
+        self[_fields.Bill.BILL_NUMBER] = int(bill_govinfo['billNumber'])
         self[_fields.Bill.DOC_CLASS_NUMBER] = bill_govinfo['suDocClassNumber']
         self[_fields.Bill.BILL_TYPE] = bill_govinfo['billType']
+        self[_fields.Bill.SESSION] = int(bill_govinfo['session'])
+        self[_fields.Bill.BILL_NUMBER] = int(bill_govinfo['billNumber'])
         self[_fields.Bill.BILL_VERSION] = bill_govinfo['billVersion']
-        self[_fields.Bill.IS_APPROPRATION] = bill_govinfo['isAppropriation']
-        self[_fields.Bill.IS_PRIVATE] = bill_govinfo['isPrivate']
-        self[_fields.Bill.COMMITTEES] = bill_govinfo['committess']
-        self[_fields.Bill.MEMBERS] = bill_govinfo['members']
+        self[_fields.Bill.IS_APPROPRATION] = \
+            bool(bill_govinfo['isAppropriation'])
+        self[_fields.Bill.IS_PRIVATE] = bool(bill_govinfo['isPrivate'])
+
+        try:
+            self[_fields.Bill.GOVERNMENT_AUTHOR] = \
+                bill_govinfo['governmentAuthor2']
+        except KeyError:
+            self[_fields.Bill.GOVERNMENT_AUTHOR] = None
+
+        try:
+            self[_fields.Bill.COMMITTEES] = bill_govinfo['committees']
+        except KeyError:
+            self[_fields.Bill.COMMITTEES] = None
+
+        try:
+            self[_fields.Bill.MEMBERS] = bill_govinfo['members']
+        except KeyError:
+            self[_fields.Bill.MEMBERS] = None
+
+    @property
+    def title(self) -> str:
+        return self[_fields.Bill.TITLE]
+
+    @property
+    def short_title(self) -> str:
+        return self[_fields.Bill.SHORT_TITLE]
+
+    @property
+    def date_issued(self) -> str:
+        return self[_fields.Bill.DATE_ISSUED]
+
+    @property
+    def number_of_pages(self) -> int:
+        return self[_fields.Bill.PAGES]
+
+    @property
+    def government_author(self) -> str:
+        return self[_fields.Bill.GOVERNMENT_AUTHOR]
+
+    @property
+    def doc_class_number(self) -> str:
+        return self[_fields.Bill.DOC_CLASS_NUMBER]
+
+    @property
+    def bill_type(self) -> str:
+        return self[_fields.Bill.BILL_TYPE]
+
+    @property
+    def session(self) -> int:
+        return self[_fields.Bill.SESSION]
+
+    @property
+    def bill_number(self) -> int:
+        return self[_fields.Bill.BILL_NUMBER]
+
+    @property
+    def bill_version(self) -> str:
+        return self[_fields.Bill.BILL_VERSION]
+
+    @property
+    def is_appropration(self) -> bool:
+        return self[_fields.Bill.IS_APPROPRATION]
+
+    @property
+    def is_private(self) -> bool:
+        return self[_fields.Bill.IS_PRIVATE]
+
+    @property
+    def committees(self) -> List:
+        return self[_fields.Bill.COMMITTEES]
+
+    @property
+    def members(self) -> List:
+        return self[_fields.Bill.MEMBERS]
 
 
 class GovInfoCongressRecord(dict):
@@ -53,7 +134,7 @@ def __init__(self, number: int, start_year: int,
 
     @property
     def number(self) -> int:
-        """The number of the given Congress0"""
+        """The number of the given Congress"""
         return self[_fields.Congress.NUMBER]
 
     @property
@@ -258,8 +339,10 @@ def _get_bills(api_key: str, congress: int):
     bill_records = []
     for package in packages:
         package_link = package['packageLink']
-        package_data = _get_text_from(package_link)
-        bill_records.append(GovInfoBillRecord(package_data))
+        package_endpoint = f'{package_link}?api_key={api_key}'
+        package_data = _get_text_from(package_endpoint)
+        package_json = _json.loads(package_data)
+        bill_records.append(GovInfoBillRecord(package_json))
 
     return bill_records
 
@@ -546,6 +629,9 @@ def _get_text_from(endpoint: str) -> str:
         try:
             response = _requests.get(_endpoint_url(endpoint))
             response_text = response.text
+
+            if response.status_code == 404:
+                raise _requests.exceptions.ConnectionError()
             break
         except _requests.exceptions.ConnectionError:
             if attempts < _util.MAX_REQUEST_ATTEMPTS:
@@ -646,7 +732,9 @@ def _granule_content_endpoint(api_key: str, package_id: str,
 
 def _endpoint_url(endpoint) -> str:
     """Creates a URL endpoint based on the path given"""
-    return _util.GOVINFO_API_URL_STR + endpoint
+    if _util.GOVINFO_API_URL_STR not in endpoint:
+        return _util.GOVINFO_API_URL_STR + endpoint
+    return endpoint
 
 
 def _query_string(**kwargs) -> str:

vistos/src/gpo/option.py

@@ -31,6 +31,18 @@ def is_valid_bioguide_state(state: Optional[str]) -> bool:
     return state in valid_states
 
 
+class Bill:
+    class Type:
+        HOUSE_BILL = 'hr'
+        SENATE_BILL = 's'
+        HOUSE_JOINT_RESOLUTION = 'hjres'
+        SENATE_JOINT_RESOLUTION = 'sjres'
+        HOUSE_CONCURRENT_RESOLUTION = 'hconres'
+        SENATE_CONCURRENT_RESOLUTION = 'sconres'
+        HOUSE_SIMPLE_RESOLUTION = 'hres'
+        SENATE_SIMPLE_RESOLUTION = 'sres'
+
+
 class Position:
     """Available options for member positions"""
     REPRESENTATIVE = 'Representative'