Merge branch 'main' into Vuln

Author: Rishi-source

.VERSION

@@ -1,3 +1,3 @@
 refs=$Format:%D$
-commit=$Format:%H$
+commit=$Format:%h$
 abbrev_commit=$Format:%H$

.dockerignore

@@ -6,7 +6,6 @@ docker-compose.yml
 
 
 # Ignore Git directory and files and github directory.
-**/.git
 **/.gitignore
 **/.gitattributes
 **/.gitmodules

.gitattributes

@@ -0,0 +1 @@
+.VERSION export-subst

.github/workflows/pypi-release.yml

@@ -47,7 +47,7 @@ jobs:
     name: Create GH release
     needs:
       - build-pypi-distribs
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Download built archives
@@ -67,7 +67,7 @@ jobs:
     name: Create PyPI release
     needs:
       - create-gh-release
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Download built archives

CHANGELOG.rst

@@ -1,6 +1,51 @@
 Release notes
 =============
 
+Version v37.0.0
+---------------------
+
+- This is a major version, this version introduces Advisory level details
+  https://github.com/aboutcode-org/vulnerablecode/issues/1796
+  https://github.com/aboutcode-org/vulnerablecode/issues/1393
+  https://github.com/aboutcode-org/vulnerablecode/issues/1883
+  https://github.com/aboutcode-org/vulnerablecode/issues/1882
+  https://github.com/aboutcode-org/vulnerablecode/pull/1866
+- We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
+- We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
+- We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- Pipeline Dashboard improvements #1920.
+- Throttle API requests based on user permissions #1909.
+- Add pipeline to compute Advisory ToDos #1764
+
+Version v36.1.3
+---------------------
+
+- Increase docker shared memory size #1896
+
+
+Version v36.1.2
+---------------------
+
+- Get tag from VERSION manifest #1895
+
+
+Version v36.1.1
+---------------------
+
+- Update is_active help text in pipeline migration #1887
+
+
+Version v36.1.0
+---------------------
+
+- Remove admin panel #1885
+- Support running pipelines in scheduled task queue #1871
+- Optimize export management command #1868
+- Fix alpine linux importer #1861
+- Stop github OSV importer crashes #1854
+- Make advisory content_id a unique field #1864
+
 
 Version v36.0.0
 ---------------------

Dockerfile

@@ -17,8 +17,21 @@ ENV PYTHONDONTWRITEBYTECODE 1
 
 RUN mkdir -p /var/vulnerablecode/static
 
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+       wait-for-it \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 # Keep the dependencies installation before the COPY of the app/ for proper caching
 COPY setup.cfg setup.py requirements.txt pyproject.toml /app/
 RUN pip install . -c requirements.txt
 
 COPY . /app
+
+# Store commit hash for docker deployment from local checkout.
+RUN if [ -d ".git" ]; then \
+  GIT_COMMIT=$(git rev-parse --short HEAD) && \
+  echo "VULNERABLECODE_GIT_COMMIT=\"$GIT_COMMIT\"" >> /app/vulnerablecode/settings.py; \
+  rm -rf .git; \
+fi

Makefile

@@ -29,6 +29,7 @@ ACTIVATE?=. ${VENV}/bin/activate;
 VIRTUALENV_PYZ=etc/thirdparty/virtualenv.pyz
 # Do not depend on Python to generate the SECRET_KEY
 GET_SECRET_KEY=`base64 /dev/urandom | head -c50`
+GET_ALTCHA_HMAC_KEY=`head -c 32 /dev/urandom | xxd -p -c 32`
 # Customize with `$ make envfile ENV_FILE=/etc/vulnerablecode/.env`
 ENV_FILE=.env
 # Customize with `$ make postgres VULNERABLECODE_DB_PASSWORD=YOUR_PASSWORD`
@@ -63,6 +64,7 @@ envfile:
 	@if test -f ${ENV_FILE}; then echo ".env file exists already"; exit 1; fi
 	@mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE}
 	@echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE}
+	@echo ALTCHA_HMAC_KEY=\"${GET_ALTCHA_HMAC_KEY}\" >> ${ENV_FILE}
 
 isort:
 	@echo "-> Apply isort changes to ensure proper imports ordering"

docker-compose.yml

@@ -9,6 +9,13 @@ services:
     volumes:
       - db_data:/var/lib/postgresql/data/
       - ./etc/postgresql/postgresql.conf:/etc/postgresql/postgresql.conf
+    shm_size: 1gb
+
+  vulnerablecode_redis:
+    image: redis
+    volumes:
+      - vulnerablecode_redis_data:/data
+    restart: always
 
   vulnerablecode:
     build: .
@@ -26,6 +33,31 @@ services:
     depends_on:
       - db
 
+  vulnerablecode_scheduler:
+    build: .
+    command: wait-for-it web:8000 -- python ./manage.py run_scheduler
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/vulnerablecode/:/etc/vulnerablecode/
+    depends_on:
+      - vulnerablecode_redis
+      - db
+      - vulnerablecode
+
+  vulnerablecode_rqworker:
+    build: .
+    command: wait-for-it web:8000 -- python ./manage.py rqworker default
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/vulnerablecode/:/etc/vulnerablecode/
+    depends_on:
+      - vulnerablecode_redis
+      - db
+      - vulnerablecode
+
+
   nginx:
     image: nginx
     ports:
@@ -44,4 +76,5 @@ services:
 volumes:
   db_data:
   static:
+  vulnerablecode_redis_data:
 

docker.env

@@ -4,3 +4,5 @@ POSTGRES_PASSWORD=vulnerablecode
 
 VULNERABLECODE_DB_HOST=db
 VULNERABLECODE_STATIC_ROOT=/var/vulnerablecode/static/
+
+VULNERABLECODE_REDIS_HOST=vulnerablecode_redis

docs/source/conf.py

@@ -38,6 +38,8 @@
     "https://example.org/api/non-existent-packages",
     "https://github.com/aboutcode-org/vulnerablecode/pull/495/commits",
     "https://nvd.nist.gov/products/cpe",
+    "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml",
+    "http://ftp.suse.com/pub/projects/security/yaml/",
 ]
 
 # Add any Sphinx extension module names here, as strings. They can be