Add retry for compatibility store (#119)

Author: liyanhui1228

badge_server/badge_server.py

@@ -410,12 +410,13 @@ def run_check():
             for py_ver in [2, 3]:
                 results = list(checker.get_pairwise_compatibility(
                     py_ver, pkg_sets))
+                logging.warning(results)
                 py_version = PY_VER_MAPPING[py_ver]
 
                 for res in results:
-                    logging.warning(res)
-                    status = res[0].get('result')
-                    package = res[0].get('packages')[1]
+                    res_item = res[0]
+                    status = res_item.get('result')
+                    package = res_item.get('packages')[1]
                     if status != 'SUCCESS':
                         # Ignore the package that not support for given py_ver
                         if package in \
@@ -424,8 +425,8 @@ def run_check():
                             continue
                         # Status showing one of the check failures
                         version_and_res[
-                            py_version]['status'] = res.get('result')
-                        description = res.get('description')
+                            py_version]['status'] = res_item.get('result')
+                        description = res_item.get('description')
                         details = EMPTY_DETAILS if description is None \
                             else description
                         version_and_res[

compatibility_lib/compatibility_lib/compatibility_store.py

@@ -17,6 +17,7 @@
 import datetime
 import enum
 import itertools
+import retrying
 from typing import Any, FrozenSet, Iterable, List, Mapping, Optional
 
 from google.cloud import bigquery
@@ -234,6 +235,8 @@ def get_self_compatibility(self,
         """
         return self.get_self_compatibilities([p])[p]
 
+    @retrying.retry(stop_max_attempt_number=7,
+                    wait_fixed=2000)
     def get_self_compatibilities(self,
                                  packages: Iterable[package.Package]) -> \
             Mapping[package.Package, List[CompatibilityResult]]:
@@ -277,6 +280,8 @@ def get_self_compatibilities(self,
         return {p: self._filter_older_versions(crs)
                 for (p, crs) in package_to_result.items()}
 
+    @retrying.retry(stop_max_attempt_number=7,
+                    wait_fixed=2000)
     def get_pair_compatibility(self, packages: List[package.Package]) -> \
             Iterable[CompatibilityResult]:
         """Returns CompatibilityStatuses for a pair of packages.
@@ -317,6 +322,8 @@ def get_pair_compatibility(self, packages: List[package.Package]) -> \
             self._row_to_compatibility_status(packages, row)
             for row in query_job)
 
+    @retrying.retry(stop_max_attempt_number=7,
+                    wait_fixed=2000)
     def get_compatibility_combinations(self,
                                        packages: List[package.Package]) -> \
             Mapping[FrozenSet[package.Package], List[CompatibilityResult]]:
@@ -409,7 +416,8 @@ def save_compatibility_statuses(
                 self._release_time_table,
                 row)
 
-
+    @retrying.retry(stop_max_attempt_number=7,
+                    wait_fixed=2000)
     def get_dependency_info(self, package_name):
         """Returns dependency info for an indicated Google OSS package.
 

dashboard/dashboard_builder.py

@@ -42,6 +42,9 @@
 from compatibility_lib import deprecated_dep_finder
 from compatibility_lib import package
 
+logging.basicConfig()
+logging.getLogger().setLevel(logging.INFO)
+
 _JINJA2_ENVIRONMENT = jinja2.Environment(
     loader=jinja2.FileSystemLoader('.'), autoescape=jinja2.select_autoescape())
 
@@ -338,7 +341,9 @@ def main():
 
     packages = [
         package.Package(install_name) for install_name in args.packages]
+    logging.info("Getting self compatibility results...")
     package_to_results = store.get_self_compatibilities(packages)
+    logging.info("Getting pairwise compatibility results...")
     pairwise_to_results = store.get_compatibility_combinations(packages)
 
     package_with_dependency_info = {}
@@ -356,15 +361,15 @@ def main():
     dashboard_builder = DashboardBuilder(packages, results)
 
     # Build the pairwise grid dashboard
-    logging.warning('Starting build the grid...')
+    logging.info('Starting build the grid...')
     grid_html = dashboard_builder.build_dashboard(
         'dashboard/grid-template.html')
     grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html'
     with open(grid_path, 'wt') as f:
         f.write(grid_html)
 
     # Build the dashboard main page
-    logging.warning('Starting build the main dashboard...')
+    logging.info('Starting build the main dashboard...')
     main_html = dashboard_builder.build_dashboard(
         'dashboard/main-template.html')