Convert sibling API event scraper classes into one class and a mixin

Author: hancush

legistar/events.py

@@ -1,4 +1,3 @@
-from abc import ABCMeta, abstractmethod
 import time
 import datetime
 from collections import deque
@@ -171,7 +170,7 @@ def _get_ecomment_link(self, link):
         return self.ecomment_dict.get(event_id, None)
 
 
-class LegistarAPIEventScraperBase(LegistarAPIScraper, metaclass=ABCMeta):
+class LegistarAPIEventScraper(LegistarAPIScraper):
     webscraper_class = LegistarEventsScraper
     WEB_RETRY_EVENTS = 3
 
@@ -197,10 +196,6 @@ def _init_webscraper(self):
 
         return webscraper
 
-    @abstractmethod
-    def _get_web_event(self, api_event):
-        pass
-
     def api_events(self, since_datetime=None):
         # scrape from oldest to newest. This makes resuming big
         # scraping jobs easier because upon a scrape failure we can
@@ -315,25 +310,6 @@ def minutes(self, event):
             self._suppress_item_matter(item, minutes_url)
             yield item
 
-    def _suppress_item_matter(self, item, agenda_url):
-        '''
-        Agenda items in Legistar do not always display links to
-        associated matter files even if the same agenda item
-        in the API references a Matter File. The agenda items
-        we scrape should honor the suppression on the Legistar
-        agendas.
-
-        This is also practical because matter files that are hidden
-        in the Legistar Agenda do not seem to available for scraping
-        on Legistar or through the API
-
-        Since we are not completely sure that the same suppression
-        logic should be used for all Legislative Bodies, this method
-        is currently just a hook for being overridden in particular
-        scrapers. As of now, at least LA Metro uses this hook.
-        '''
-        pass
-
     def rollcalls(self, event):
         for item in self.agenda(event):
             if item['EventItemRollCallFlag']:
@@ -354,6 +330,25 @@ def addDocs(self, e, events, doc_type):
         except ValueError:
             pass
 
+    def _suppress_item_matter(self, item, agenda_url):
+        '''
+        Agenda items in Legistar do not always display links to
+        associated matter files even if the same agenda item
+        in the API references a Matter File. The agenda items
+        we scrape should honor the suppression on the Legistar
+        agendas.
+
+        This is also practical because matter files that are hidden
+        in the Legistar Agenda do not seem to available for scraping
+        on Legistar or through the API
+
+        Since we are not completely sure that the same suppression
+        logic should be used for all Legislative Bodies, this method
+        is currently just a hook for being overridden in particular
+        scrapers. As of now, at least LA Metro uses this hook.
+        '''
+        pass
+
     def _event_status(self, event):
         '''Events can have a status of tentative, confirmed, cancelled, or
         passed (http://docs.opencivicdata.org/en/latest/data/event.html). By
@@ -367,11 +362,20 @@ def _event_status(self, event):
             status = 'confirmed'
 
         return status
-
-
-class LegistarAPIEventScraper(LegistarAPIEventScraperBase):
+    
+    def _not_in_web_interface(self, event):
+        '''Occasionally, an event will appear in the API before it appears in
+        the web interface. This method checks attributes of the API event that
+        tell uswhether the given event is one of those cases, returning True if
+        so, and False otherwise. Available for override in jurisdictional
+        scrapers.
+        '''
+        return False
 
     def _get_web_event(self, api_event):
+        if self._not_in_web_interface(api_event):
+            return None
+        
         return self.web_detail(api_event)
 
     def web_detail(self, event):
@@ -405,12 +409,13 @@ def web_detail(self, event):
         return event_page_details
 
 
-class LegistarAPIEventScraperZip(LegistarAPIEventScraperBase):
-    '''
-    There are some inSite sites that have information that only appears
-    event listing page, like NYC's 'Meeting Topic.' This scraper visits
-    the listing page and attempts to zip API and web events together
-    '''
+class WebCalendarMixin:
+    """
+    Sometimes, it's desirable to retrieve information from the web calendar,
+    in addition to the API. This mixin extends the base functionality to get
+    event information from both the detail page, if accessible, and the web
+    calendar listing.
+    """
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -422,13 +427,20 @@ def __init__(self, *args, **kwargs):
         # Instantiate dictionary where events from generator are stored as they
         # are scraped.
         self._scraped_events = {}
-
+        
     def _get_web_event(self, api_event):
         if self._not_in_web_interface(api_event):
-            return None
+            event_detail = {}
         else:
-            # None if entire web calendar scraped but API event not found
-            return self.web_results(api_event)
+            # None if detail link does not exist or cannot be found.
+            event_detail = super()._get_web_event(api_event) or {}
+        
+        # Sometimes events can appear on the calendar before their detail links
+        # become available. None if entire web calendar scraped but event not
+        # found.
+        event_listing = self.web_results(api_event) or {}
+        
+        return (event_listing | event_detail) or None
 
     def web_results(self, event):
         api_key = (event['EventBodyName'].strip(),
@@ -453,29 +465,22 @@ def _scrapeWebCalendar(self):
         chronological order.
         '''
         for event, _ in self._webscraper.events(follow_links=False):
-            event_key = self._event_key(event, self._webscraper)
+            event_key = self._event_key(event)
+            print(event_key, event)
             yield event_key, event
 
-    def _event_key(self, event, web_scraper):
+    def _event_key(self, event):
         '''Since Legistar InSite contains more information about events than
         are available in the API, we need to scrape both. Then, we have
         to line them up. This method makes a key that should be
         uniquely identify every event and will allow us to link
         events from the two data sources.
         '''
-        response = web_scraper.get(event['iCalendar']['url'], verify=False)
-        event_time = web_scraper.ical(response.text).subcomponents[0]['DTSTART'].dt
+        response = self._webscraper.get(event['iCalendar']['url'], verify=False)
+        event_time = self._webscraper.ical(response.text).subcomponents[0]['DTSTART'].dt
         event_time = pytz.timezone(self.TIMEZONE).localize(event_time)
 
         key = (event['Name']['label'],
                event_time)
 
         return key
-
-    def _not_in_web_interface(self, event):
-        '''Occasionally, an event will appear in the API, but not in the web
-        interface. This method checks attributes of the API event that tell us
-        whether the given event is one of those cases, returning True if so, and
-        False otherwise. Available for override in jurisdictional scrapers.
-        '''
-        return False