Skip to content
This repository was archived by the owner on Oct 21, 2022. It is now read-only.

Commit 898abdd

Browse files
brianquinlanbrianquinlan
authored
Update dependencies using a single insert_rows. (#227)
1 parent d88bb53 commit 898abdd

2 files changed

Lines changed: 195 additions & 29 deletions

File tree

‎compatibility_lib/compatibility_lib/compatibility_store.py‎

Lines changed: 72 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ def _compatibility_status_to_row(
179179
return row
180180

181181
@staticmethod
182-
def _compatibility_status_to_release_time_row(
182+
def _compatibility_status_to_release_time_rows(
183183
cs: CompatibilityResult) -> List[Mapping[str, Any]]:
184184
"""Converts a CompatibilityResult into a dict which is a row for
185185
release time table."""
@@ -403,43 +403,86 @@ def save_compatibility_statuses(
403403
self._pairwise_table,
404404
pair_rows)
405405

406-
release_time_rows = {}
406+
# Dependencies are not stored per Python version. This is not
407+
# theoretically sound but is probably good enough in practice.
408+
#
409+
# If there are multiple compatibility results for the same package,
410+
# use the dependencies with the highest version for that package.
411+
# For example, if the following CompatibilityResults were passed to
412+
# `save_compatibility_statuses`:
413+
#
414+
# cr1 = CompatibilityResult(
415+
# packages=[Package('package1')],
416+
# dependency_info={'package1': {'installed_version': '1.2.3' ...})
417+
# cr2 = CompatibilityResult(
418+
# packages=[Package('package1')],
419+
# dependency_info={'package1': {'installed_version': '1.2.4' ...})
420+
#
421+
# then the dependency information for `cr2` would be saved because it
422+
# is the newest version ('1.2.4' vs '1.2.3'). If the versions are the
423+
# same then choose one arbitrarily.
424+
#
425+
# This check is done to prevent an old versions of apache-beam, which
426+
# was accidentally released for Python 3, from having it's dependencies
427+
# stored. It will also make sure that the Python 3 version of package
428+
# dependencies are stored when Python 2 releases stop happening.
429+
install_name_to_compatibility_result = {}
407430
for cs in compatibility_statuses:
408431
if len(cs.packages) == 1:
409432
install_name = cs.packages[0].install_name
410-
# Only store the dep info for latest version of the package
411-
# being checked. e.g. pip install apache-beam will have
412-
# different version installed in py2/3.
413-
if not self._should_update_dep_info(
414-
cs, release_time_rows.get(install_name)):
415-
continue
416-
row = self._compatibility_status_to_release_time_row(cs)
417-
if row:
418-
release_time_rows[install_name] = row
419-
420-
for row in release_time_rows.values():
433+
if install_name not in install_name_to_compatibility_result:
434+
install_name_to_compatibility_result[install_name] = cs
435+
else:
436+
old_version = self._get_package_version(
437+
install_name_to_compatibility_result[install_name])
438+
new_version = self._get_package_version(cs)
439+
# TODO: Do not compare versions lexicographically.
440+
# Lexicographically, '10' < '9'.
441+
if new_version > old_version:
442+
install_name_to_compatibility_result[install_name] = cs
443+
444+
dependency_rows = itertools.chain(
445+
*[self._compatibility_status_to_release_time_rows(cs)
446+
for cs in install_name_to_compatibility_result.values()])
447+
448+
# Insert the dependency rows in a stable order to make testing more
449+
# convenient.
450+
dependency_rows = sorted(
451+
dependency_rows,
452+
key=lambda row: (row['install_name'], row['dep_name']))
453+
454+
if dependency_rows:
421455
self._client.insert_rows(
422456
self._release_time_table,
423-
row)
457+
dependency_rows)
424458

425-
def _should_update_dep_info(self, cs, dep_info_stored):
426-
"""Return True if the stored version is behind latest version."""
427-
if dep_info_stored is None:
428-
return True
459+
def _get_package_version(self, result: CompatibilityResult) -> str:
460+
"""Returns the version of the single package in a CompatibilityResult.
429461
430-
install_name = cs.packages[0].install_name
431-
install_name_sanitized = install_name.split('[')[0]
432-
installed_version = cs.dependency_info[
433-
install_name_sanitized]['installed_version']
462+
Args:
463+
result: The compatibility result. This result must contain exactly
464+
one package.
465+
466+
Returns:
467+
A string containing the version of the single package found in the
468+
CompatibilityResult `packages` attribute. For example:
469+
470+
cr1 = CompatibilityResult(
471+
packages=[Package('package1')],
472+
dependency_info={'package1': {'installed_version': '1.2.3' ..})
473+
_get_package_version(cr1) => '1.2.3'
474+
"""
475+
if len(result.packages) != 1:
476+
raise ValueError('multiple packages found in CompatibilityResult')
434477

435-
installed_version_stored = '0'
436-
for row in dep_info_stored:
437-
if row['install_name'] == install_name \
438-
and row['dep_name'] == install_name_sanitized:
439-
installed_version_stored = row['installed_version']
440-
break
478+
install_name = result.packages[0].install_name
479+
install_name_sanitized = install_name.split('[')[0]
441480

442-
return True if installed_version > installed_version_stored else False
481+
for pkg, version_info in result.dependency_info.items():
482+
if pkg == install_name_sanitized:
483+
return version_info['installed_version']
484+
raise ValueError('missing version information for {}'.format(
485+
install_name_sanitized))
443486

444487
@retrying.retry(stop_max_attempt_number=7,
445488
wait_fixed=2000)

‎compatibility_lib/compatibility_lib/test_compatibility_store.py‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,129 @@ def MockClient(project=None):
473473
mock_client.insert_rows.assert_called_with(
474474
store._release_time_table, [row_release_time])
475475

476+
def test_save_compatibility_statuses_release_time_for_latest_many_packages(
477+
self):
478+
mock_client = mock.Mock()
479+
timestamp = '2018-07-17 03:01:06.11693 UTC'
480+
status = compatibility_store.Status.SUCCESS
481+
apache_beam_py2 = mock.Mock(
482+
packages=[package.Package('apache-beam[gcp]')],
483+
python_major_version='2',
484+
status=status,
485+
details=None,
486+
dependency_info={
487+
'six': {
488+
'installed_version': '9.9.9',
489+
'installed_version_time': '2018-05-12T16:26:31',
490+
'latest_version': '2.7.0',
491+
'current_time': '2018-07-13T17:11:29.140608',
492+
'latest_version_time': '2018-05-12T16:26:31',
493+
'is_latest': False,
494+
} ,
495+
'apache-beam': {
496+
'installed_version': '2.7.0',
497+
'installed_version_time': '2018-05-12T16:26:31',
498+
'latest_version': '2.7.0',
499+
'current_time': '2018-07-13T17:11:29.140608',
500+
'latest_version_time': '2018-05-12T16:26:31',
501+
'is_latest': True,
502+
}},
503+
timestamp=timestamp)
504+
apache_beam_py3 = mock.Mock(
505+
packages=[package.Package('apache-beam[gcp]')],
506+
python_major_version='3',
507+
status=status,
508+
details=None,
509+
dependency_info={'apache-beam': {
510+
'installed_version': '2.2.0',
511+
'installed_version_time': '2018-05-12T16:26:31',
512+
'latest_version': '2.7.0',
513+
'current_time': '2018-07-13T17:11:29.140608',
514+
'latest_version_time': '2018-05-12T16:26:31',
515+
'is_latest': False,
516+
}},
517+
timestamp=timestamp)
518+
google_api_core_py2 = mock.Mock(
519+
packages=[package.Package('google-api-core')],
520+
python_major_version='2',
521+
status=status,
522+
details=None,
523+
dependency_info={
524+
'google-api-core': {
525+
'installed_version': '3.7.0',
526+
'installed_version_time': '2018-05-12T16:26:31',
527+
'latest_version': '2.7.0',
528+
'current_time': '2018-07-13T17:11:29.140608',
529+
'latest_version_time': '2018-05-12T16:26:31',
530+
'is_latest': True,
531+
}},
532+
timestamp=timestamp)
533+
google_api_core_py3 = mock.Mock(
534+
packages=[package.Package('google-api-core')],
535+
python_major_version='3',
536+
status=status,
537+
details=None,
538+
dependency_info={'google-api-core': {
539+
'installed_version': '3.7.1',
540+
'installed_version_time': '2018-05-12T16:26:31',
541+
'latest_version': '2.7.0',
542+
'current_time': '2018-07-13T17:11:29.140608',
543+
'latest_version_time': '2018-05-12T16:26:31',
544+
'is_latest': False,
545+
}},
546+
timestamp=timestamp)
547+
548+
apache_beam_row = {
549+
'install_name': 'apache-beam[gcp]',
550+
'dep_name': 'apache-beam',
551+
'installed_version': '2.7.0',
552+
'installed_version_time': '2018-05-12T16:26:31',
553+
'latest_version': '2.7.0',
554+
'latest_version_time': '2018-05-12T16:26:31',
555+
'is_latest': True,
556+
'timestamp': '2018-07-13T17:11:29.140608',
557+
}
558+
559+
six_row = {
560+
'install_name': 'apache-beam[gcp]',
561+
'dep_name': 'six',
562+
'installed_version': '9.9.9',
563+
'installed_version_time': '2018-05-12T16:26:31',
564+
'latest_version': '2.7.0',
565+
'latest_version_time': '2018-05-12T16:26:31',
566+
'is_latest': False,
567+
'timestamp': '2018-07-13T17:11:29.140608',
568+
}
569+
570+
google_api_core_row = {
571+
'install_name': 'google-api-core',
572+
'dep_name': 'google-api-core',
573+
'installed_version': '3.7.1',
574+
'installed_version_time': '2018-05-12T16:26:31',
575+
'latest_version': '2.7.0',
576+
'latest_version_time': '2018-05-12T16:26:31',
577+
'is_latest': False,
578+
'timestamp': '2018-07-13T17:11:29.140608',
579+
}
580+
581+
def MockClient(project=None):
582+
return mock_client
583+
584+
patch_client = mock.patch(
585+
'compatibility_lib.compatibility_store.bigquery.Client',
586+
MockClient)
587+
588+
with patch_client:
589+
store = compatibility_store.CompatibilityStore()
590+
store.save_compatibility_statuses(
591+
[apache_beam_py2,
592+
apache_beam_py3,
593+
google_api_core_py2,
594+
google_api_core_py3])
595+
596+
mock_client.insert_rows.assert_called_with(
597+
store._release_time_table,
598+
[apache_beam_row, six_row, google_api_core_row])
476599

477600
class MockClient(object):
478601

0 commit comments

Comments
 (0)