Bump dependencies

.dockerignore

@@ -3,5 +3,6 @@
 local.ini
 source_info.json
 ./import_data/
+./dumps/
 
 #END OF FILE

.gitignore

@@ -145,6 +145,7 @@ settings.ini
 # folder where data are git cloned
 ./import_data
 import_data
+./dumps
 
 # Virtual Studio Code settings
 .vscode

README.md

@@ -113,16 +113,16 @@ docker compose -f docker-compose.yaml -f docker-compose.dev.yaml down --rmi all
 
 ## How to do a dump
 ```sh
-docker exec -e PGPASSWORD=the_super_password $(docker ps -q) pg_dump --clean -h localhost  -U postgres --format plain | sed "s/pbkdf2_sha256[^\t]*/redacted/g" > my_dump.sql
+docker exec -e PGPASSWORD=the_super_password ifbcatsrc_db_1 pg_dump --clean -h localhost  -U postgres --format plain | sed "s/pbkdf2_sha256[^\t]*/redacted/g" > my_dump.sql
 ```
 
 
-## How to retore a db dump
+## How to restore a db dump
 We consider here that no container are started. You have to get the dump, and uncompress it in the root directory of the project, and name it `data`
 ```sh
 docker stop $(docker ps -q)
-docker compose -f docker-compose.yaml -f docker-compose.dev.yaml run -d db 
-docker exec -e PGPASSWORD=the_super_password $(docker ps -q) psql -h localhost -U postgres -f /code/data
+docker compose -f docker-compose.yaml -f docker-compose.import.yaml run -d db 
+docker exec $(docker ps -q) psql -h localhost -U postgres -f /code/dumps/latest.sql
 ```
 
 # How to manage the server

django.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-bullseye
+FROM python:3.12
 
 RUN apt-get update && \
     apt-get install -y \
@@ -10,18 +10,20 @@ RUN apt-get update && \
  && rm -rf /var/lib/apt/lists/* \
  && python -m pip install --upgrade pip
 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 EXPOSE 8000
 RUN mkdir /code
 WORKDIR /code
-CMD ["gunicorn", "--reload", "--reload-engine", "inotify", "--chdir", "ifbcat", "--bind", ":8000", "ifbcat.wsgi:application"]
+CMD ["gunicorn", "--bind", ":8000", "ifbcat.asgi:application", "-k", "uvicorn.workers.UvicornWorker"]
 
 COPY requirements.txt /code/
 RUN python -m pip install --upgrade pip && pip install -r requirements.txt
 
 COPY ./resources/*-entrypoint.sh /
 RUN chmod a+x /*-entrypoint.sh
 
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
 COPY . /code/
 
 ARG CI_COMMIT_SHA

docker-compose.dev.yaml

@@ -1,23 +1,44 @@
-version: '3'
-
 services:
   db:
     volumes:
       - .:/code # for dev purpose only !!!
-      - ifbcat-dev-db-data:/var/lib/postgresql/data
+      - ifbcat-dev-db-data:/var/lib/postgresql
 
   web:
+    command:
+      - "gunicorn"
+      - "--reload"
+      - "--reload-engine"
+      - "inotify"
+      - "--chdir"
+      - "ifbcat"
+      - "--bind"
+      - ":8000"
+      - "ifbcat.asgi:application"
+      - "-k"
+      - "uvicorn.workers.UvicornWorker"
+    environment:
+     - NGINX_FROWARD_PROTO=http
+     - POSTGRES_PASSWORD=the_super_password
+     - DJANGO_DEBUG=True
+     - IS_DEV_INSTANCE=True
     volumes:
       - .:/code # for dev purpose only !!!
       - ./import_data:/code/import_data # for dev purpose only !!!
 
 
+  nginx:
+    environment:
+     - NGINX_FROWARD_PROTO=http
+
+
+
   # The Database manager (adminer)
-  adminer:
-    image: adminer
-    restart: always
-    ports:
-      - "8081:8080"
+#  adminer:
+#    image: adminer
+#    restart: always
+#    ports:
+#      - "8081:8080"
 
 volumes:
   ifbcat-dev-db-data:

docker-compose.import.yaml

@@ -1,6 +1,8 @@
-version: '3'
-
 services:
+  db:
+    volumes:
+      - .:/code # for dev purpose only !!!
+
   web:
     restart:
       on-failure

docker-compose.yaml

@@ -1,11 +1,9 @@
-version: '3'
-
 services:
 
   db:
-    image: postgres:12
+    image: postgres:18
     volumes:
-      - /var/ifbcat/db_data:/var/lib/postgresql/data
+      - /var/ifbcat/db_data:/var/lib/postgresql
     restart: always
     ports:
       - "5433:5432"
@@ -19,11 +17,9 @@ services:
       dockerfile: django.dockerfile
     restart:
       always
-    entrypoint: /docker-entrypoint.sh
-    command: gunicorn --reload ifbcat.wsgi -b 0.0.0.0:8000 --threads 2
     environment:
-      - CI_COMMIT_SHA=$CI_COMMIT_SHA
-      - CI_COMMIT_DATE=$CI_COMMIT_DATE
+      - CI_COMMIT_SHA=${CI_COMMIT_SHA:-xx}
+      - CI_COMMIT_DATE=${CI_COMMIT_DATE:-xx}
     env_file:
       - ./resources/default.ini
       - ./local.ini
@@ -37,21 +33,11 @@ services:
   nginx:
     image: nginx:1.19-alpine
     volumes:
-      - ./resources/nginx.conf:/etc/nginx/conf.d/default.conf:ro
+      - ./resources/nginx.conf.template:/etc/nginx/templates/default.conf.template:ro
       - /var/ifbcat/static:/static:ro
     ports:
       - "8080:8080"
-    depends_on:
-      - web
-
-  nginx-https:
-    image: nginx:1.19-alpine
-    volumes:
-      - ./resources/nginx-https.conf:/etc/nginx/conf.d/default.conf:ro
-      - /var/ifbcat/static:/static:ro
-      - /etc/ssl/certs/wildcard-chained-france-bioinformatique.crt:/wildcard-chained-france-bioinformatique.crt:ro
-      - /etc/ssl/private/wildcard-france-bioinformatique.key:/wildcard-france-bioinformatique.key:ro
-    ports:
-      - "443:8443"
+    environment:
+     - NGINX_FROWARD_PROTO=https
     depends_on:
       - web

ifbcat/asgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for ifbcat project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ifbcat.settings')
+
+application = get_asgi_application()

ifbcat/settings.example.ini

@@ -1,3 +1,4 @@
 [settings]
 POSTGRES_PASSWORD=the_super_password
-DEBUG=True
+DJANGO_DEBUG=True
+IS_DEV_INSTANCE=True

ifbcat/settings.py

@@ -26,7 +26,9 @@
 SECRET_KEY = config('SECRET_KEY', '#1eg!-*r&1*y8*s9!g^!if!-(1&11k0%*7b$-jwgv!7u!ae7wt')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = config('DEBUG', 'true').lower() == 'true'
+DEBUG = config('DJANGO_DEBUG', 'true').lower() == 'true'
+
+IS_DEV_INSTANCE = config('IS_DEV_INSTANCE', 'false').lower() == 'true'
 
 try:
     ALLOWED_HOSTS = [s.strip() for s in config('ALLOWED_HOSTS', '127.0.0.1').split(',')]

ifbcat_api/admin.py

@@ -306,7 +306,7 @@ def _action(modeladmin, request, qset):
                     messages.SUCCESS,
                 )
 
-        name = 'revoke_%s' % re.sub('[\W]+', '_', str(group))
+        name = 'revoke_%s' % re.sub(r'[\W]+', '_', str(group))
         return name, (_action, name, 'Revoke permissions "%s"' % group)
 
     @staticmethod
@@ -380,7 +380,7 @@ def _action(modeladmin, request, qset):
                     messages.SUCCESS,
                 )
 
-        name = 'grant_%s' % re.sub('[\W]+', '_', str(group))
+        name = 'grant_%s' % re.sub(r'[\W]+', '_', str(group))
         return name, (_action, name, 'Grant permissions "%s"' % group)
 
     @staticmethod

ifbcat_api/misc.py

@@ -7,6 +7,49 @@
 from django.db.models import ManyToManyRel, ManyToOneRel
 from opencage.geocoder import OpenCageGeocode
 from rest_framework import serializers
+import os
+import json
+import functools
+from typing import Any, Callable
+
+
+def disk_cache(func: Callable) -> Callable:
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs) -> Any:
+        cache_base = os.environ.get('CACHE_DIR')
+        if not cache_base:
+            return func(*args, **kwargs)
+        cache_dir = os.path.join(cache_base, func.__name__)
+        os.makedirs(cache_dir, exist_ok=True)
+
+        a_args = args[1:] if args and isinstance(args[0], type) else args
+        arg_str = "_".join(repr(arg) for arg in a_args)
+        kwarg_str = "_".join(f"{k}={repr(v)}" for k, v in sorted(kwargs.items()))
+        key = f"{arg_str}_{kwarg_str}"
+        key = "".join(c if c.isalnum() or (c in "-_.") else "_" for c in key)
+        key = key.strip("_")
+        file_path = os.path.join(cache_dir, f"{key}.json")
+
+        # Try loading from cache
+        try:
+            with open(file_path, 'r') as f:
+                return json.load(f)
+        except FileNotFoundError:
+            pass
+
+        # Call the actual function
+        result = func(*args, **kwargs)
+
+        # Save result to cache
+        try:
+            with open(file_path, 'w') as f:
+                json.dump(result, f)
+        except Exception as e:
+            print(f"Failed to write cache {file_path}: {e}")
+
+        return result
+
+    return wrapper
 
 
 class BibliographicalEntryNotFound(Exception):

ifbcat_api/model/misc.py

@@ -101,6 +101,15 @@ def get_permission_classes(cls):
             IsAuthenticatedOrReadOnly,
         )
 
+    @classmethod
+    def get_edam_info_ebi_ols_url(cls, uri):
+        return f'https://www.ebi.ac.uk/ols4/api/ontologies/edam/terms?iri={uri}'
+
+    @classmethod
+    @misc.disk_cache
+    def get_edam_from_ebi_ols(cls, uri):
+        return requests.get(cls.get_edam_info_ebi_ols_url(uri)).json()
+
     def update_information_from_ebi_ols(self):
         response = misc.get_edam_info_from_ols(self.uri)
         try:
@@ -306,6 +315,7 @@ def get_permission_classes(cls):
         )
 
     @classmethod
+    @misc.disk_cache
     def get_doi_from_pmid(cls, pmid):
         with Entrez.efetch(db="pubmed", id=str(pmid), rettype="xml", retmode="text") as handle:
             d = Entrez.read(handle)

ifbcat_api/static/css/ifbcat_admin.css

@@ -285,4 +285,7 @@ html, body{
 #jazzy-tabs{
     margin-bottom: 0!important;
 }
+.dev-instance .content-wrapper{
+    background-color: orange !important;
+}
 /**/

ifbcat_api/tests/test_no_views_crash.py

@@ -64,7 +64,7 @@ def test_dashboard(self):
         self.assertEqual(self.client.get('/admin/').status_code, 200)
 
 
-class TestNoViewsCrash(EnsureImportDataAreHere):
+class TestCaseWithData(EnsureImportDataAreHere):
     def setUp(self):
         super().setUp()
         # load the whole catalogue
@@ -79,6 +79,8 @@ def setUp(self):
         add_everywhere(f)
         logger.warning('Data loaded')
 
+
+class TestNoViewsCrash(TestCaseWithData):
     def test_all_at_once_to_spare_resource(self):
         #######################################################################
         # test_loadcatalog

ifbcat_api/tests/test_sitemap.py

@@ -0,0 +1,45 @@
+import re
+from collections import defaultdict
+
+from django.urls import reverse
+
+from ifbcat_api import models
+from ifbcat_api.tests.test_no_views_crash import TestCaseWithData
+
+
+class TestApi(TestCaseWithData):
+    by_passed_links = {
+        'http://testserver/api/team/Pasteur%20HUB/',
+    }
+    link_count_to_test = 10
+
+    def test_create_new_file(self):
+        response = self.client.get(reverse('sitemap-general'))
+        resp = response.content.decode()
+        url_pattern = r'<loc>(.*?)</loc>'
+        links = re.findall(url_pattern, resp)
+        grouped_links = defaultdict(set)
+
+        for link in links:
+            group = link.find('/', link.find('api') + 4)
+            grouped_links[group].add(link)
+
+        total = 0
+        for links in grouped_links.values():
+            links_to_test = list(links)[: self.link_count_to_test]
+            links_to_test.extend(self.by_passed_links & links)
+
+            for link in links_to_test:
+                self.assertEqual(self.client.get(link, follow=True).status_code, 200, link)
+                total += 1
+        self.assertGreater(total, 0)
+
+
+class TestApiCustomInstance(TestApi):
+    link_count_to_test = 1000
+
+    def setUp(self):
+        # don't call super().setUp(), create instance instead, can only be an instance in the sitemap
+        models.Team.objects.get_or_create(
+            name="foo",
+        )

ifbcat_api/urls.py

@@ -56,7 +56,7 @@
     # path('testapiview/', views.TestApiView.as_view()),
     path('tool/<int:pk>/', views.ToolViewSet.as_view({'get': 'retrieve'})),
     path('tool/<biotoolsID>/', views.ToolViewSet.as_view({'get': 'retrieve'})),
-    re_path('topic/(?P<uri>topic_\d+)/', views.TopicViewSet.as_view({'get': 'retrieve'})),
+    re_path(r'topic/(?P<uri>topic_\d+)/', views.TopicViewSet.as_view({'get': 'retrieve'})),
     path('login/', views.UserLoginApiView.as_view()),
     path('', include(router.urls)),
     path(