Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
93d6704
feat: move Vanir signature generation to a cron job
cuixq Mar 10, 2026
d6f7ff0
lint
cuixq Mar 25, 2026
ef3574d
worker test and format
cuixq Mar 25, 2026
7146c37
Merge branch 'master' into vanir
cuixq Mar 25, 2026
41cbff1
uncomment
cuixq Mar 25, 2026
62b8b00
worker test
cuixq Mar 25, 2026
d029187
Merge branch 'master' into vanir
cuixq Mar 25, 2026
6ae2f17
gitter port for vanir test
cuixq Mar 25, 2026
ac72b2f
remove vanir test
cuixq Mar 26, 2026
7679d1c
add vanir test back and increase timeout
cuixq Mar 26, 2026
707f3ad
increase timeout further
cuixq Mar 26, 2026
f255fe5
debug setup gitter
cuixq Mar 26, 2026
29271cf
Merge branch 'master' into vanir
cuixq Mar 26, 2026
e6f7f9f
batch
cuixq Mar 27, 2026
5b186e8
unused import
cuixq Mar 30, 2026
510e27b
Merge branch 'master' into vanir
cuixq Mar 30, 2026
9b830a9
cron last successful time
cuixq Mar 30, 2026
b094052
no bug but vulnerability
cuixq Mar 30, 2026
3d91b7b
Merge branch 'master' into vanir
cuixq Mar 30, 2026
fd7ac8a
gcs
cuixq Mar 30, 2026
b465213
revert
cuixq Mar 30, 2026
3822a45
max worker
cuixq Mar 31, 2026
bf5263e
retry list
cuixq Mar 31, 2026
574e24c
gcs
cuixq Mar 31, 2026
bb8b11f
comment
cuixq Mar 31, 2026
e3377cc
Merge branch 'master' into vanir
cuixq Mar 31, 2026
abbe641
Merge branch 'master' into vanir
cuixq Apr 1, 2026
8f36957
skip vanir generation and add timestamp
cuixq Apr 2, 2026
386562d
Merge branch 'master' into vanir
cuixq Apr 2, 2026
3b7e9bc
format
cuixq Apr 2, 2026
672cfdd
timestamp name
cuixq Apr 2, 2026
34ada66
naming
cuixq Apr 2, 2026
3609f71
format
cuixq Apr 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ importer-tests:
recoverer-tests:
cd gcp/workers/recoverer && ./run_tests.sh

vanir-signatures-tests:
cd gcp/workers/vanir_signatures && ./run_tests.sh

website-tests:
cd gcp/website && ./run_tests.sh

Expand Down
12 changes: 12 additions & 0 deletions deployment/build-and-stage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,16 @@ steps:
args: ['push', '--all-tags', 'gcr.io/oss-vdb-test/osv-linter']
waitFor: ['build-osv-linter', 'cloud-build-queue']

# Build/push vanir-signatures images to gcr.io/oss-vdb.
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'gcr.io/oss-vdb/vanir-signatures:latest', '-t', 'gcr.io/oss-vdb/vanir-signatures:$COMMIT_SHA', '.']
dir: 'gcp/workers/vanir_signatures'
id: 'build-vanir-signatures'
waitFor: ['build-worker']
- name: gcr.io/cloud-builders/docker
args: ['push', '--all-tags', 'gcr.io/oss-vdb/vanir-signatures']
waitFor: ['build-vanir-signatures', 'cloud-build-queue']

# Build/push cron job images.
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'gcr.io/oss-vdb/cron:latest', '-t', 'gcr.io/oss-vdb/cron:$COMMIT_SHA', '.']
Expand Down Expand Up @@ -432,6 +442,7 @@ steps:
relations=gcr.io/oss-vdb/relations:$COMMIT_SHA,\
generatesitemap=gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA,\
gitter=gcr.io/oss-vdb/gitter:$COMMIT_SHA,\
vanir-signatures=gcr.io/oss-vdb/vanir-signatures:$COMMIT_SHA,\
cron=gcr.io/oss-vdb/cron:$COMMIT_SHA"
]
dir: deployment/clouddeploy/gke-workers
Expand Down Expand Up @@ -496,3 +507,4 @@ images:
- 'gcr.io/oss-vdb/oss-fuzz-importer:$COMMIT_SHA'
- 'gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA'
- 'gcr.io/oss-vdb/gitter:$COMMIT_SHA'
- 'gcr.io/oss-vdb/vanir-signatures:$COMMIT_SHA'
1 change: 1 addition & 0 deletions deployment/clouddeploy/gke-workers/base/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,5 @@ resources:
- record-checker.yaml
- cve5-to-osv.yaml
- custommetrics.yaml
- vanir-signatures.yaml

26 changes: 26 additions & 0 deletions deployment/clouddeploy/gke-workers/base/vanir-signatures.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: vanir-signatures
labels:
cronLastSuccessfulTimeMins: "1500"
spec:
schedule: "0 9 * * *"
timeZone: "Australia/Sydney"
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
spec:
containers:
- name: vanir-signatures
image: vanir-signatures
imagePullPolicy: Always
resources:
requests:
cpu: "1"
memory: "10G"
limits:
cpu: "1"
memory: "13G"
restartPolicy: Never
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ patches:
- path: record-checker.yaml
- path: custommetrics.yaml
- path: gitter.yaml
- path: vanir-signatures.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: vanir-signatures
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: vanir-signatures
env:
- name: GOOGLE_CLOUD_PROJECT
value: oss-vdb-test
- name: OSV_VULNERABILITIES_BUCKET
value: osv-test-vulnerabilities
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,5 @@ patches:
- path: cve5-to-osv.yaml
- path: custommetrics.yaml
- path: gitter.yaml
- path: vanir-signatures.yaml

Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: vanir-signatures
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: vanir-signatures
env:
- name: GOOGLE_CLOUD_PROJECT
value: oss-vdb
- name: OSV_VULNERABILITIES_BUCKET
value: osv-vulnerabilities
8 changes: 8 additions & 0 deletions gcp/workers/cloudbuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ steps:
- GITTER_PORT=8891
waitFor: ['init', 'sync']

- name: 'gcr.io/oss-vdb/ci'
id: 'vanir-signatures-tests'
dir: gcp/workers/vanir_signatures
args: ['bash', '-ex', 'run_tests.sh']
env:
- DATASTORE_EMULATOR_PORT=8006
waitFor: ['init', 'sync']

timeout: 7200s
options:
machineType: E2_HIGHCPU_8
Expand Down
41 changes: 0 additions & 41 deletions gcp/workers/oss_fuzz_worker/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
from google.cloud import pubsub_v1
from google.cloud import storage
from google.cloud.storage import retry
from google.protobuf import json_format

sys.path.append(os.path.dirname(os.path.realpath(__file__)))
import osv
Expand All @@ -42,8 +41,6 @@
from osv import vulnerability_pb2
import oss_fuzz

from vanir import vulnerability_manager

DEFAULT_WORK_DIR = '/work'
OSS_FUZZ_GIT_URL = 'https://github.com/google/oss-fuzz.git'
TASK_SUBSCRIPTION = 'oss-fuzz-tasks'
Expand Down Expand Up @@ -499,43 +496,6 @@ def _analyze_vulnerability(self, source_repo, repo, vulnerability, path,
vulnerability.id)
raise UpdateConflictError

def _generate_vanir_signatures(self, vulnerability):
"""Generates Vanir signatures for a vulnerability."""
if not any(r.type == vulnerability_pb2.Range.GIT
for affected in vulnerability.affected
for r in affected.ranges):
logging.info(
'Skipping Vanir signature generation for %s as it has no '
'GIT affected ranges.', vulnerability.id)
return vulnerability
if any(affected.package.name == "Kernel" and
affected.package.ecosystem == "Linux"
for affected in vulnerability.affected):
logging.info(
'Skipping Vanir signature generation for %s as it is a '
'Kernel vulnerability.', vulnerability.id)
return vulnerability

logging.info('Generating Vanir signatures for %s', vulnerability.id)
try:
vuln_manager = vulnerability_manager.generate_from_json_string(
content=json.dumps([
json_format.MessageToDict(
vulnerability, preserving_proto_field_name=True)
]),)
vuln_manager.generate_signatures()

if not vuln_manager.vulnerabilities:
logging.warning('Vanir signature generation resulted in no '
'vulnerabilities.')
return vulnerability

return vuln_manager.vulnerabilities[0].to_proto()
except Exception:
logging.exception('Failed to generate Vanir signatures for %s',
vulnerability.id)
return vulnerability

def _do_update(self, source_repo, repo, vulnerability, relative_path,
original_sha256):
"""Process updates on a vulnerability."""
Expand All @@ -552,7 +512,6 @@ def _do_update(self, source_repo, repo, vulnerability, relative_path,
orig_modified_date = vulnerability.modified.ToDatetime(datetime.UTC)

# Fully enrich the vulnerability object in memory.
vulnerability = self._generate_vanir_signatures(vulnerability)
try:
result = self._analyze_vulnerability(source_repo, repo, vulnerability,
relative_path, original_sha256)
Expand Down
19 changes: 19 additions & 0 deletions gcp/workers/vanir_signatures/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM gcr.io/oss-vdb/worker

COPY vanir_signatures.py /usr/local/bin/vanir_signatures.py
RUN chmod 755 /usr/local/bin/vanir_signatures.py
ENTRYPOINT ["vanir_signatures.py"]
22 changes: 22 additions & 0 deletions gcp/workers/vanir_signatures/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash -ex
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

cd ../worker

# Install dependencies only if not running in Cloud Build
if [ -z "$CLOUDBUILD" ]; then
poetry sync
fi
poetry run python ../vanir_signatures/vanir_signatures_test.py
Loading
Loading