Cache the github head version result based on commit number (#201)

Author: liyanhui1228

badge_server/main.py

@@ -39,7 +39,8 @@
 
 def _get_result_from_cache(
         package_name: str,
-        badge_type: badge_utils.BadgeType) -> dict:
+        badge_type: badge_utils.BadgeType,
+        commit_number: str = None) -> dict:
     """Get check result from cache."""
     # Return unknown if package not in whitelist
     if not utils._is_package_in_whitelist([package_name]):
@@ -49,7 +50,9 @@ def _get_result_from_cache(
             details={})
     # Get the result from cache, return None if not in cache
     else:
-        result = cache.get('{}_{}'.format(package_name, badge_type.value))
+        package_key = '{}_{}'.format(
+            package_name, commit_number) if commit_number else package_name
+        result = cache.get('{}_{}'.format(package_key, badge_type.value))
 
     if result is None:
         result = badge_utils._build_default_result(
@@ -109,7 +112,7 @@ def _get_pair_status_for_packages(pkg_sets):
     return version_and_res
 
 
-def _get_all_results_from_cache(package_name):
+def _get_all_results_from_cache(package_name, commit_number=None):
     """Get all the check results from cache.
 
     Rules:
@@ -118,13 +121,16 @@ def _get_all_results_from_cache(package_name):
     """
     self_compat_res = _get_result_from_cache(
         package_name=package_name,
-        badge_type=badge_utils.BadgeType.SELF_COMP_BADGE)
+        badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
+        commit_number=commit_number)
     google_compat_res = _get_result_from_cache(
         package_name=package_name,
-        badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE)
+        badge_type=badge_utils.BadgeType.GOOGLE_COMP_BADGE,
+        commit_number=commit_number)
     dependency_res = _get_result_from_cache(
         package_name=package_name,
-        badge_type=badge_utils.BadgeType.DEP_BADGE)
+        badge_type=badge_utils.BadgeType.DEP_BADGE,
+        commit_number=commit_number)
 
     if self_compat_res['py3']['status'] == 'SUCCESS' and \
         google_compat_res['py3']['status'] == 'SUCCESS' and \
@@ -177,6 +183,8 @@ def one_badge_image():
         badge_name = badge_utils.GITHUB_HEAD_NAME
         is_github = True
 
+    commit_number = badge_utils._calculate_commit_number(package_name)
+
     force_run_check = flask.request.args.get('force_run_check')
     # Remove the last '/' from the url root
     url_prefix = flask.request.url_root[:-1]
@@ -186,19 +194,23 @@ def one_badge_image():
     requests.get(url_prefix + flask.url_for(
         'self_compatibility_badge_image',
         package=package_name,
-        force_run_check=force_run_check))
+        force_run_check=force_run_check,
+        commit_number=commit_number))
     # Google compatibility badge
     requests.get(url_prefix + flask.url_for(
         'google_compatibility_badge_image',
         package=package_name,
-        force_run_check=force_run_check))
+        force_run_check=force_run_check,
+        commit_number=commit_number))
     # Self dependency badge
     requests.get(url_prefix + flask.url_for(
         'self_dependency_badge_image',
         package=package_name,
-        force_run_check=force_run_check))
+        force_run_check=force_run_check,
+        commit_number=commit_number))
 
-    status, timestamp, _, _, _ = _get_all_results_from_cache(package_name)
+    status, timestamp, _, _, _ = _get_all_results_from_cache(
+        package_name, commit_number=commit_number)
     color = badge_utils.STATUS_COLOR_MAPPING[status]
 
     details_link = url_prefix + flask.url_for('one_badge_target',
@@ -224,24 +236,30 @@ def one_badge_image():
 @app.route('/one_badge_target')
 def one_badge_target():
     package_name = flask.request.args.get('package')
+    commit_number = badge_utils._calculate_commit_number(package_name)
+
     status, _, self_compat_res, google_compat_res, dependency_res = \
-        _get_all_results_from_cache(package_name)
+        _get_all_results_from_cache(package_name, commit_number)
 
     return flask.render_template(
         'one-badge.html',
         package_name=package_name,
         self_compat_res=self_compat_res,
         google_compat_res=google_compat_res,
-        dependency_res=dependency_res)
+        dependency_res=dependency_res,
+        commit_number=commit_number)
 
 
 @app.route('/self_compatibility_badge_image')
 def self_compatibility_badge_image():
     """Badge showing whether a package is compatible with itself."""
     package_name = flask.request.args.get('package')
     force_run_check = flask.request.args.get('force_run_check')
+    commit_number = flask.request.args.get('commit_number')
 
     badge_name = flask.request.args.get('badge_name')
+    package_key = '{}_{}'.format(
+        package_name, commit_number) if commit_number else package_name
 
     if badge_name is None:
         badge_name = 'self compatibility'
@@ -285,16 +303,15 @@ def run_check():
             badge_utils.TIMESTAMP_FORMAT)
 
         # Write the result to Cloud Datastore
-        cache.set(
-            '{}_self_comp_badge'.format(package_name), version_and_res)
+        cache.set('{}_self_comp_badge'.format(package_key), version_and_res)
 
     if not utils._is_package_in_whitelist([package_name]):
         self_comp_res = badge_utils._build_default_result(
             badge_type=badge_utils.BadgeType.SELF_COMP_BADGE,
             status='UNKNOWN',
             details=badge_utils.PACKAGE_NOT_SUPPORTED)
     else:
-        self_comp_res = cache.get('{}_self_comp_badge'.format(package_name))
+        self_comp_res = cache.get('{}_self_comp_badge'.format(package_key))
 
     if self_comp_res is None:
         details = version_and_res
@@ -347,7 +364,11 @@ def self_dependency_badge_image():
 
     package_name = flask.request.args.get('package')
     force_run_check = flask.request.args.get('force_run_check')
+    commit_number = flask.request.args.get('commit_number')
+
     badge_name = flask.request.args.get('badge_name')
+    package_key = '{}_{}'.format(
+        package_name, commit_number) if commit_number else package_name
 
     if badge_name is None:
         badge_name = 'dependency status'
@@ -382,17 +403,15 @@ def run_check():
             badge_utils.TIMESTAMP_FORMAT)
 
         # Write the result to Cloud Datastore
-        cache.set(
-            '{}_dependency_badge'.format(package_name), res)
+        cache.set('{}_dependency_badge'.format(package_key), res)
 
     if not utils._is_package_in_whitelist([package_name]):
         dependency_res = badge_utils._build_default_result(
             badge_type=badge_utils.BadgeType.DEP_BADGE,
             status='UNKNOWN',
             details={})
     else:
-        dependency_res = cache.get(
-            '{}_dependency_badge'.format(package_name))
+        dependency_res = cache.get('{}_dependency_badge'.format(package_key))
 
     if dependency_res is None:
         details = badge_utils.DEFAULT_DEPENDENCY_RESULT
@@ -433,7 +452,11 @@ def google_compatibility_badge_image():
     to one of the failure types, details can be found at the target link."""
     package_name = flask.request.args.get('package')
     force_run_check = flask.request.args.get('force_run_check')
+    commit_number = flask.request.args.get('commit_number')
+
     badge_name = flask.request.args.get('badge_name')
+    package_key = '{}_{}'.format(
+        package_name, commit_number) if commit_number else package_name
 
     if badge_name is None:
         badge_name = 'google compatibility'
@@ -492,11 +515,9 @@ def run_check():
             result = version_and_res
 
         # Write the result to Cloud Datastore
-        cache.set(
-            '{}_google_comp_badge'.format(package_name), result)
+        cache.set('{}_google_comp_badge'.format(package_key), result)
 
-    google_comp_res = cache.get(
-        '{}_google_comp_badge'.format(package_name))
+    google_comp_res = cache.get('{}_google_comp_badge'.format(package_key))
 
     if not utils._is_package_in_whitelist([package_name]):
         google_comp_res = badge_utils._build_default_result(

badge_server/templates/one-badge.html

@@ -30,6 +30,10 @@
                         <div class="col-md-12">
                             <h1 class="title-4">Package name: {{ package_name }}
                             </h1>
+                            {% if commit_number | length > 0 %}
+                                <h1 class="title-4">Commit number: {{ commit_number }}
+                                </h1>
+                            {% endif %}
                             <hr class="line-seprate">
                         </div>
                     </div>

badge_server/utils.py

@@ -15,7 +15,12 @@
 """Common utils methods for badge server."""
 
 import enum
+import json
+import logging
 import os
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
 from typing import Optional
 
 import pybadges
@@ -39,6 +44,7 @@
 
 URL_PREFIX = 'https://img.shields.io/badge/'
 GITHUB_HEAD_NAME = 'github head'
+GITHUB_API = 'https://api.github.com/repos'
 SVG_CONTENT_TYPE = 'image/svg+xml'
 EMPTY_DETAILS = 'NO DETAILS'
 PACKAGE_NOT_SUPPORTED = "The package is not supported by checker server."
@@ -141,3 +147,26 @@ def _get_badge(res: dict, badge_name: str) -> str:
         left_text=badge_name,
         right_text=status,
         right_color=color)
+
+
+def _calculate_commit_number(package: str) -> Optional[str]:
+    """Calculate the github head version commit number."""
+    url_parsed = urlparse(package)
+    if url_parsed.scheme and url_parsed.netloc == 'github.com':
+        try:
+            owner, repo, *_ = url_parsed.path[1:].split('/')
+            repo = repo.split('.git')[0]
+        except ValueError:
+            return None
+        else:
+            url = '{0}/{1}/{2}/commits'.format(GITHUB_API, owner, repo)
+            try:
+                with urlopen(url) as f:
+                    commits = json.loads(f.read())
+                return commits[0]['sha']
+            except Exception as e:
+                logging.warning(
+                    'Unable to generate caching key for "%s": %s', package, e)
+                return None
+
+    return None

compatibility_lib/compatibility_lib/get_compatibility_data.py

@@ -53,25 +53,11 @@ def _result_dict_to_compatibility_result(results, python_version):
     return res_list
 
 
-def _generate_pairs_for_github_head():
-    """Generate pairs for each github head package with the PyPI packages.
-
-    e.g. [(github_pkg, pkg1), (github_pkg, pkg2),...]
-    """
-    pkg_pairs = []
-
-    for gh_pkg in configs.WHITELIST_URLS.keys():
-        gh_pairs = [(gh_pkg, package) for package in configs.PKG_LIST]
-        pkg_pairs.extend(gh_pairs)
-
-    return pkg_pairs
-
-
 def write_to_status_table():
-    """Get the compatibility status for PyPI and github head versions."""
+    """Get the compatibility status for PyPI versions."""
     # Write self compatibility status to BigQuery
     self_res_list = []
-    packages = configs.PKG_LIST + list(configs.WHITELIST_URLS.keys())
+    packages = configs.PKG_LIST
     for py_version in [PY2, PY3]:
         results = checker.get_self_compatibility(
             python_version=py_version,
@@ -88,14 +74,6 @@ def write_to_status_table():
         res_list = _result_dict_to_compatibility_result(results, py_version)
         store.save_compatibility_statuses(res_list)
 
-        # For github head versions
-        pkg_sets = _generate_pairs_for_github_head()
-        results = checker.get_pairwise_compatibility(
-            python_version=py_version,
-            pkg_sets=pkg_sets)
-        res_list = _result_dict_to_compatibility_result(results, py_version)
-        store.save_compatibility_statuses(res_list)
-
 
 if __name__ == '__main__':
     write_to_status_table()

compatibility_lib/compatibility_lib/test_get_compatibility_data.py

@@ -72,30 +72,6 @@ def mock_init():
             'compatibility_lib.get_compatibility_data.store',
             self.fake_store)
 
-    def test__generate_pairs_for_github_head(self):
-        pkg_list = ['package1', 'package2']
-        gh_pkgs = {
-            'gh_pkg1_url': 'gh_pkg1',
-            'gh_pkg2_url':'gh_pkg2'
-        }
-        patch_pkg_list = mock.patch(
-            'compatibility_lib.configs.PKG_LIST', pkg_list)
-        patch_gh_list = mock.patch(
-            'compatibility_lib.configs.WHITELIST_URLS', gh_pkgs)
-
-        with self.patch_constructor, self.patch_checker, self.patch_store,\
-             patch_pkg_list, patch_gh_list:
-            from compatibility_lib import get_compatibility_data
-
-            pairs = get_compatibility_data._generate_pairs_for_github_head()
-
-        expected = [('gh_pkg1_url', 'package1'),
-                    ('gh_pkg1_url', 'package2'),
-                    ('gh_pkg2_url', 'package1'),
-                    ('gh_pkg2_url', 'package2')]
-
-        self.assertEqual(set(pairs), set(expected))
-
     def test__result_dict_to_compatibility_result(self):
         with self.patch_constructor, self.patch_checker, self.patch_store:
             from compatibility_lib import compatibility_store
@@ -123,7 +99,7 @@ def test_write_to_status_table(self):
         saved_results = self.fake_store._packages_to_compatibility_result.get(
             frozenset({self.packages[0]}))
         self.assertIsNotNone(saved_results)
-        self.assertEqual(len(saved_results), 6)
+        self.assertEqual(len(saved_results), 4)
         saved_item = saved_results[0]
         self.assertEqual(saved_item.packages, self.packages)
         self.assertEqual(saved_item.dependency_info, self.dependency_info)