T50264 filter relative delta

README.md

@@ -9,31 +9,14 @@ Code Yellow backend framework for SPA webapps with REST-like API.
 
 ## Running the tests
 
-There are two ways to run the tests:
-- Run directly `./setup.py test` (requires you to have python3 and postgres installed)
-- Run with docker `docker-compose run binder ./setup.py test`
-  - Access the test database directly by with `docker-compose run db psql -h db -U postgres`.
-  - It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker-compose rm`. The database will be created during the setup in `tests/__init__.py`.
+Run with docker `docker compose run binder ./setup.py test` (but you may need to run `docker compose build db binder` first)
+- Access the test database directly by with `docker compose run db psql -h db -U postgres`.
+- It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker compose rm`. The database will be created during the setup in `tests/__init__.py`.
 
 The tests are set up in such a way that there is no need to keep migration files. The setup procedure in `tests/__init__.py` handles the preparation of the database by directly calling some build-in Django commands.
 
 To only run a selection of the tests, use the `-s` flag like `./setup.py test -s tests.test_some_specific_test`.
 
 ## MySQL support
 
-MySQL is supported, but only with the goal to replace it with
-PostgreSQL.  This means it has a few limitations:
-
-- `where` filtering on `with` relations is not supported.
-- Only integer primary keys are supported.
-- When fetching large number of records using `with` or the ids are big, be sure to increase `GROUP_CONCAT` max string length by:
-
-```
-DATABASES = {
-	'default': {
-		'OPTIONS': {
-            'init_command': 'SET SESSION group_concat_max_len = 1000000',
-        },
-	},
-}
-```
+MySQL was supported at some point, but not anymore I guess.

binder/json.py

@@ -27,7 +27,7 @@
 # dateutil.relativedelta serializer, if available
 try:
 	from dateutil.relativedelta import relativedelta
-	from relativedeltafield import format_relativedelta
+	from relativedeltafield.utils import format_relativedelta
 	SERIALIZERS[relativedelta] = format_relativedelta
 except ImportError:
 	pass

binder/models.py

@@ -321,6 +321,25 @@ def clean_qualifier(self, qualifier, value):
 		return qualifier, cleaned_value
 
 
+class RelativeDeltaFieldFilter(FieldFilter):
+	name = 'RelativeDeltaFieldFilter'
+	fields = []
+	allowed_qualifiers = [None, 'in', 'gt', 'gte', 'lt', 'lte', 'range', 'isnull']
+
+	def clean_value(self, qualifier, v):
+		from relativedeltafield.utils import parse_relativedelta
+		try:
+			return parse_relativedelta(v)
+		except ValueError:
+			raise ValidationError(v + ' is not a valid (extended) ISO8601 interval specification')
+
+try:
+	from relativedeltafield import RelativeDeltaField
+	RelativeDeltaFieldFilter.fields.append(RelativeDeltaField)
+except ImportError:
+	pass
+
+
 class TimeFieldFilter(FieldFilter):
 	fields = [models.TimeField]
 	# Maybe allow __startswith? And __year etc?

ci-requirements.txt

@@ -9,3 +9,5 @@ coverage
 django-hijack<3.0.0
 openpyxl
 pika
+python-dateutil >= 2.6.0
+django-relativedelta >= 2.0.0

docker-compose.yml

@@ -1,6 +1,8 @@
 services:
   db:
-    image: postgres:11.5
+    image: postgres:13.21
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust # Insecure, but fine for just for running tests.
   binder:
     build: .
     command: tail -f /dev/null

setup.py

@@ -44,6 +44,7 @@
 		'pika == 1.3.2',
 	],
 	tests_require=[
+		'django-relativedelta >= 2.0.0',
 		'django-hijack >= 2.1.10, < 3.0.0',
 		(
 			'mysqlclient >= 1.3.12'

tests/__init__.py

@@ -31,105 +31,105 @@
 		'USER': 'postgres',
 	}
 
-settings.configure(**{
-	'DEBUG': True,
-	'SECRET_KEY': 'testy mctestface',
-	'ALLOWED_HOSTS': ['*'],
-	'DATABASES': {
-		'default': db_settings,
-	},
-	'MIDDLEWARE': [
-		# TODO: Try to reduce the set of absolutely required middlewares
-		'request_id.middleware.RequestIdMiddleware',
-		'django.contrib.sessions.middleware.SessionMiddleware',
-		'django.middleware.csrf.CsrfViewMiddleware',
-		'django.contrib.auth.middleware.AuthenticationMiddleware',
-		'binder.plugins.token_auth.middleware.TokenAuthMiddleware',
-	],
-	'INSTALLED_APPS': [
-		# TODO: Try to reduce the set of absolutely required applications
-		'django.contrib.auth',
-		'django.contrib.contenttypes',
-		'django.contrib.sessions',
-		'binder',
-		'binder.plugins.token_auth',
-		'tests',
-		'tests.testapp',
-	],
-	'MIGRATION_MODULES': {
-		'testapp': None,
-		'auth': None,
-		'sessions': None,
-		'contenttypes': None,
-		'binder': None,
-		'token_auth': None,
-	},
-	'USE_TZ': True,
-	'TIME_ZONE': 'UTC',
-	'ROOT_URLCONF': 'tests.testapp.urls',
-	'LOGGING': {
-		'version': 1,
-		'handlers': {
-			'console': {
-				'level': 'DEBUG',
-				'class': 'logging.StreamHandler',
-			},
+if not settings.configured:
+	settings.configure(**{
+		'DEBUG': True,
+		'SECRET_KEY': 'testy mctestface',
+		'ALLOWED_HOSTS': ['*'],
+		'DATABASES': {
+			'default': db_settings,
 		},
-		'loggers': {
-			# We override only this one to avoid logspam
-			# while running tests.  Django warnings are
-			# stil shown.
-			'binder': {
-				'handlers': ['console'],
-				'level': 'ERROR',
-			},
-		}
-	},
-	'BINDER_PERMISSION': {
-		'default': [
-			('auth.reset_password_user', None),
-			('auth.view_user', 'own'),
-			('auth.activate_user', None),
-			('auth.unmasquerade_user', None),  # If you are masquarade, the user must be able to unmasquarade
-			('auth.login_user', None),
-			('auth.signup_user', None),
-			('auth.logout_user', None),
-			('auth.change_own_password_user', None),
+		'MIDDLEWARE': [
+			# TODO: Try to reduce the set of absolutely required middlewares
+			'request_id.middleware.RequestIdMiddleware',
+			'django.contrib.sessions.middleware.SessionMiddleware',
+			'django.middleware.csrf.CsrfViewMiddleware',
+			'django.contrib.auth.middleware.AuthenticationMiddleware',
+			'binder.plugins.token_auth.middleware.TokenAuthMiddleware',
 		],
-		# Basic permissions which can be used to override stuff
-		'testapp.view_country': [
-
-		]
-	},
-	'GROUP_PERMISSIONS': {
-		'admin': [
-			'testapp.view_country'
-		]
-	},
-	'GROUP_CONTAINS': {
-		'admin': []
-	},
-	'INTERNAL_MEDIA_HEADER': 'X-Accel-Redirect',
-	'INTERNAL_MEDIA_LOCATION': '/internal/media/',
-})
+		'INSTALLED_APPS': [
+			# TODO: Try to reduce the set of absolutely required applications
+			'django.contrib.auth',
+			'django.contrib.contenttypes',
+			'django.contrib.sessions',
+			'binder',
+			'binder.plugins.token_auth',
+			'tests',
+			'tests.testapp',
+		],
+		'MIGRATION_MODULES': {
+			'testapp': None,
+			'auth': None,
+			'sessions': None,
+			'contenttypes': None,
+			'binder': None,
+			'token_auth': None,
+		},
+		'USE_TZ': True,
+		'TIME_ZONE': 'UTC',
+		'ROOT_URLCONF': 'tests.testapp.urls',
+		'LOGGING': {
+			'version': 1,
+			'handlers': {
+				'console': {
+					'level': 'DEBUG',
+					'class': 'logging.StreamHandler',
+				},
+			},
+			'loggers': {
+				# We override only this one to avoid logspam
+				# while running tests.  Django warnings are
+				# stil shown.
+				'binder': {
+					'handlers': ['console'],
+					'level': 'ERROR',
+				},
+			}
+		},
+		'BINDER_PERMISSION': {
+			'default': [
+				('auth.reset_password_user', None),
+				('auth.view_user', 'own'),
+				('auth.activate_user', None),
+				('auth.unmasquerade_user', None),  # If you are masquarade, the user must be able to unmasquarade
+				('auth.login_user', None),
+				('auth.signup_user', None),
+				('auth.logout_user', None),
+				('auth.change_own_password_user', None),
+			],
+			# Basic permissions which can be used to override stuff
+			'testapp.view_country': [
 
-setup()
+			]
+		},
+		'GROUP_PERMISSIONS': {
+			'admin': [
+				'testapp.view_country'
+			]
+		},
+		'GROUP_CONTAINS': {
+			'admin': []
+		},
+		'INTERNAL_MEDIA_HEADER': 'X-Accel-Redirect',
+		'INTERNAL_MEDIA_LOCATION': '/internal/media/',
+	})
+	setup()
 
-# Do the dance to ensure the models are synched to the DB.
-# This saves us from having to include migrations
-from django.core.management.commands.migrate import Command as MigrationCommand # noqa
-from django.db import connections # noqa
-from django.db.migrations.executor import MigrationExecutor # noqa
+	# Do the dance to ensure the models are synched to the DB.
+	# This saves us from having to include migrations
+	from django.core.management.commands.migrate import Command as MigrationCommand # noqa
+	from django.db import connections # noqa
+	from django.db.migrations.executor import MigrationExecutor # noqa
 
-# This is oh so hacky....
-cmd = MigrationCommand()
-cmd.verbosity = 0
-connection = connections['default']
-executor = MigrationExecutor(connection)
-cmd.sync_apps(connection, executor.loader.unmigrated_apps)
+	# This is oh so hacky....
+	cmd = MigrationCommand()
+	cmd.verbosity = 0
+	connection = connections['default']
+	executor = MigrationExecutor(connection)
+	cmd.sync_apps(connection, executor.loader.unmigrated_apps)
 
-# Hack to make the view_country permission, which doesn't work with the MigrationCommand somehow
-from django.contrib.auth.models import Group, Permission, ContentType
-content_type = ContentType.objects.get_or_create(app_label='testapp', model='country')[0]
-Permission.objects.get_or_create(content_type=content_type, codename='view_country')
-call_command('define_groups')
+	# Hack to make the view_country permission, which doesn't work with the MigrationCommand somehow
+	from django.contrib.auth.models import Group, Permission, ContentType
+	content_type = ContentType.objects.get_or_create(app_label='testapp', model='country')[0]
+	Permission.objects.get_or_create(content_type=content_type, codename='view_country')
+	call_command('define_groups')

tests/plugins/test_loaded_values_mixin.py

@@ -1,3 +1,4 @@
+from dateutil.relativedelta import relativedelta
 from django.test import TestCase
 from ..testapp.models import Animal, Zoo, Caretaker, ZooEmployee
 
@@ -37,6 +38,7 @@ def test_old_values_after_initialization_are_identical_to_current_but_unchanged(
 			'zoo_of_birth': artis.id,
 			'caretaker': caretaker.id,
 			'deleted': False,
+			'feeding_period': relativedelta(days=1),
 			'birth_date': None,
 		}, scooby.get_old_values())
 
@@ -77,6 +79,7 @@ def test_old_values_after_change_are_marked_as_changed_and_old_values_returns_ol
 			'zoo_of_birth': artis.id,
 			'caretaker': None,
 			'deleted': False,
+			'feeding_period': relativedelta(days=1),
 			'birth_date': None,
 		}, scooby.get_old_values())
 
@@ -112,6 +115,7 @@ def test_old_values_return_current_value_after_fresh_fetch_from_db(self):
 			'zoo_of_birth': artis.id,
 			'caretaker': None,
 			'deleted': False,
+			'feeding_period': relativedelta(days=1),
 			'birth_date': None,
 		}, scooby.get_old_values())
 
@@ -150,6 +154,7 @@ def test_recursion_depth_issue_with_loaded_values_and_only(self):
 			'zoo_of_birth': artis.id,
 			'caretaker': None,
 			'deleted': False,
+			'feeding_period': None,
 			'birth_date': None,
 		}, scooby.get_old_values())
 

tests/test_filter_relative_delta.py

@@ -0,0 +1,55 @@
+from dateutil.relativedelta import relativedelta
+from django.contrib.auth.models import User
+from django.test import Client, TestCase
+from json import loads
+from .testapp.models import Animal
+
+
+class FilterRelativeDeltaTest(TestCase):
+    def setUp(self):
+        super().setUp()
+        u = User(username='testuser', is_active=True, is_superuser=True)
+        u.set_password('test')
+        u.save()
+        self.client = Client()
+        r = self.client.login(username='testuser', password='test')
+        self.assertTrue(r)
+
+    def test_filter(self):
+        bokito = Animal.objects.create(name='Bokito', feeding_period=relativedelta(hours=5))
+        harambe = Animal.objects.create(name='Harambe', feeding_period=relativedelta(days=2))
+        self.assertEqual(bokito.id, Animal.objects.filter(feeding_period__lt=relativedelta(days=1)).get().id)
+        response = self.client.get('/animal/?.feeding_period:gt=P1DT6H')
+        content = loads(response.content)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual(1, len(content['data']))
+        self.assertEqual(harambe.id, content['data'][0]['id'])
+
+    def test_sort(self):
+        bokito = Animal.objects.create(name='Bokito', feeding_period=relativedelta(hours=5))
+        harambe = Animal.objects.create(name='Harambe', feeding_period=relativedelta(days=2))
+        otto = Animal.objects.create(name='Otto', feeding_period=relativedelta(days=1, hours=1))
+
+        sanity = list(Animal.objects.all().order_by('feeding_period'))
+        self.assertEqual(bokito.id, sanity[0].id)
+        self.assertEqual(otto.id, sanity[1].id)
+        self.assertEqual(harambe.id, sanity[2].id)
+
+        response = self.client.get('/animal/?order_by=feeding_period')
+        content = loads(response.content)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual(3, len(content['data']))
+        self.assertEqual(bokito.id, content['data'][0]['id'])
+        self.assertEqual(otto.id, content['data'][1]['id'])
+        self.assertEqual(harambe.id, content['data'][2]['id'])
+
+        response = self.client.get('/animal/?order_by=-feeding_period')
+        content = loads(response.content)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual(3, len(content['data']))
+        self.assertEqual(harambe.id, content['data'][0]['id'])
+        self.assertEqual(otto.id, content['data'][1]['id'])
+        self.assertEqual(bokito.id, content['data'][2]['id'])
+
+    def test_default_value(self):
+        self.assertEqual(relativedelta(days=1), Animal.objects.create(name='Default').feeding_period)