Merges function sections

.flake8

@@ -0,0 +1,3 @@
+# Autoformatter friendly flake8 config (all formatting rules disabled)
+[flake8]
+extend-ignore = D1, D2, E1, E2, E3, E501, W1, W2, W3, W5

.github/workflows/build.yml

@@ -1,11 +1,11 @@
 name: Build
-
+permissions: {}
 on:
   workflow_dispatch: {}
 
   push:
     branches:
-      - "master"
+      - master
 
     tags:
       - "*"
@@ -21,16 +21,16 @@ jobs:
 
     steps:
       - name: Clone polytracker repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and export
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           tags: trailofbits/polytracker:latest
@@ -57,7 +57,17 @@ jobs:
       - name: Load image
         run: docker load --input /tmp/polytracker.tar
 
-      - name: Run tests
+      - name: C++ (write side) tests
+        uses: addnab/docker-run-action@v3
+        with:
+          image: trailofbits/polytracker:latest
+          run: |
+            cd /polytracker-build/unittests/src/taintdag && \
+            pwd && \
+            chmod +x tests-taintdag && \
+            ./tests-taintdag
+
+      - name: Python (integration) tests
         uses: addnab/docker-run-action@v3
         with:
           image: trailofbits/polytracker:latest
@@ -73,7 +83,7 @@ jobs:
 
     steps:
       - name: Clone polytracker repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
@@ -88,10 +98,10 @@ jobs:
         run: docker load --input /tmp/polytracker.tar
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build ${{ matrix.example }}
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: examples
           file: examples/Dockerfile-${{ matrix.example }}.demo

.gitignore

@@ -16,4 +16,5 @@ __pycache__
 /Default/
 polytracker.egg-info
 /polytracker.egg-info
-compile_commands.json
+compile_commands.json
+/test_inputs/*

.trunk/.gitignore

@@ -6,3 +6,4 @@
 plugins
 user_trunk.yaml
 user.yaml
+tmp

.trunk/trunk.yaml

@@ -1,44 +1,56 @@
 version: 0.1
 cli:
-  version: 1.14.1
+  version: 1.22.11
 plugins:
   sources:
     - id: trunk
-      ref: v1.2.1
+      ref: v1.6.7
       uri: https://github.com/trunk-io/plugins
 runtimes:
   enabled:
-    - go@1.19.5
-    - node@18.12.1
+    - go@1.21.0
+    - node@18.20.5
     - python@3.10.8
 lint:
+  threshold:
+    - linters: [ALL]
+      level: high
   ignore:
     - linters: [ALL]
       paths:
         - polytracker/src/compiler-rt/**
+        - third_party/**
+    - linters: [bandit]
+      paths:
+        - tests/**
+  definitions:
+    - name: flake8
+      direct_configs:
+        - .flake8
+
   enabled:
-    - taplo@0.8.1
-    - actionlint@1.6.25
-    - bandit@1.7.5
-    - black@23.7.0
-    - checkov@2.4.5
+    - taplo@0.9.3
+    - actionlint@1.7.7
+    - bandit@1.8.3
+    - black@25.1.0
+    - checkov@3.2.390
     - clang-format@16.0.3
-    - flake8@6.1.0
+    - flake8@7.1.2
     - git-diff-check
-    - hadolint@2.12.0
-    - isort@5.12.0
-    - markdownlint@0.35.0
-    - mypy@1.5.1
-    - oxipng@8.0.0
-    - prettier@3.0.2
-    - ruff@0.0.285
-    - shellcheck@0.9.0
+    - hadolint@2.12.1-beta
+    - isort@6.0.1
+    - markdownlint@0.44.0
+    - mypy@1.15.0
+    - oxipng@9.1.4
+    - prettier@3.5.3
+    - ruff@0.11.1
+    - shellcheck@0.10.0
     - shfmt@3.6.0
-    - svgo@3.0.2
-    - terrascan@1.18.3
-    - trivy@0.44.1
-    - trufflehog@3.48.0
-    - yamllint@1.32.0
+    - svgo@3.3.2
+    - terrascan@1.19.1
+    - trivy@0.60.0
+    - trufflehog@3.88.18
+    - yamllint@1.36.2
 actions:
   disabled:
     - trunk-announce

CMakeLists.txt

@@ -22,13 +22,13 @@ endif()
 
 set(CMAKE_CXX_STANDARD 20)
 
-add_subdirectory(third_party/Catch2)
-list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/third_party/Catch2/contrib")
 add_subdirectory(third_party/indicators)
 set(SPDLOG_NO_EXCEPTIONS TRUE)
 add_subdirectory(third_party/spdlog)
 
 add_subdirectory(polytracker)
 
 enable_testing()
+add_subdirectory(third_party/Catch2)
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/third_party/Catch2/contrib")
 add_subdirectory(unittests/src/taintdag)

Dockerfile

@@ -1,5 +1,5 @@
 # Build base image
-FROM ubuntu:jammy as base
+FROM ubuntu:jammy AS base
 
 LABEL org.opencontainers.image.authors="evan.sultanik@trailofbits.com"
 
@@ -41,7 +41,7 @@ RUN GO111MODULE=off go get github.com/SRI-CSL/gllvm/cmd/...
 ENV PATH=$PATH:/root/go/bin
 
 # Clone llvm to build `libc++` from source
-FROM base as llvm-sources
+FROM base AS llvm-sources
 
 RUN git clone --depth 1 --branch llvmorg-13.0.0 https://github.com/llvm/llvm-project.git /llvm-project
 
@@ -54,7 +54,7 @@ RUN git clone --depth 1 --branch llvmorg-13.0.0 https://github.com/llvm/llvm-pro
 
 # Build "clean" `libc++` with `gclang`. Used to link the uninstrumented
 # target of the user project. Installed into `/cxx_lib/clean_build`.
-FROM llvm-sources as clean-libcxx
+FROM llvm-sources AS clean-libcxx
 
 ENV WLLVM_BC_STORE=/cxx_clean_bitcode
 RUN mkdir -p $WLLVM_BC_STORE
@@ -78,7 +78,7 @@ RUN cmake --build $LIBCXX_BUILD_DIR --target install-cxx install-cxxabi -j$((`np
 
 # Build "poly" `libc++` with `gclang`. Used to link the instrumented
 # target of the user project. Installed into `/cxx_lib/poly_build`.
-FROM clean-libcxx as poly-libcxx
+FROM clean-libcxx AS poly-libcxx
 
 ENV WLLVM_BC_STORE=/cxx_poly_bitcode
 RUN mkdir -p $WLLVM_BC_STORE
@@ -104,7 +104,7 @@ RUN cmake -GNinja \
 RUN cmake --build $LIBCXX_BUILD_DIR --target install-cxx install-cxxabi -j$((`nproc`+1))
 
 # Build and install the polytracker
-FROM poly-libcxx as polytracker
+FROM poly-libcxx AS polytracker
 
 ARG DFSAN_FILENAME_ARCH=x86_64
 

docs/tdag.md

@@ -37,10 +37,9 @@ Some specifics:
 - [Sources](../polytracker/src/taint_sources/taint_sources.cpp) contains source labels (byte offsets into the input)
 - The Source Label Index is a bitmap that defines how to index the sources section.
 - [Sinks](../polytracker/include/taintdag/sink.h) contains sink labels (representing bytes of the output)
-- [Strings](../polytracker/include/taintdag/string_table.h) todo(kaoudis) the string table is used in conjunction with the fnmapping to put together an earlier version of the control flow log used for grammar extraction
-- [Functions](../polytracker/include/taintdag/fnmapping.h) todo(kaoudis) this contains an early version of the function list part of the control flow log used for grammar extraction
-- [Events](../polytracker/include/taintdag/fntrace.h) todo(kaoudis) this contains an early version of the entry and exit events used to structure the control flow log
-- [Control Flow Log](../polytracker/include/taintdag/control_flow_log.h): this consists of the function entry and exit records we need to reconstruct the call stack that data flow passed through.
+- [Strings](../polytracker/include/taintdag/string_table.h) the string table is a section with two current purposes: it contains names of sources, and also the function names used with the functions section to map to the control flow log. Its design is general purpose: it can store any type of stringly data we eventually decide to store.
+- [Functions](../polytracker/include/taintdag/fnmapping.h) todo(kaoudis) this section maps the low-level function identifiers used in the cflog to the actual (mangled) names in the strings table. see tests/test_cf_log.py for how this layout looks/works.
+- [Control Flow Log](../polytracker/include/taintdag/control_flow_log.h): this consists of the function entry and exit records we need to reconstruct the call stack that data flow ("tainted control flow" or "control affecting data flow" are ways that we have referred to this subset of data flow) passed through, plus recorded function ids that can be mapped back through the Functions section to the String Table to get mangled symbols. see test_cf_log.py and unittests/src/taintdag/tdag.cpp for details of how this looks/works.
 
 ## TDAG Contents
 

examples/Dockerfile-acropalypse.demo

@@ -27,5 +27,5 @@ RUN CPPFLAGS="-I$(pwd)/zlib-1.2.13/include" LDFLAGS="-L$(pwd)/zlib-1.2.13/lib" p
 RUN polytracker extract-bc -o pngtest.bc pngtest
 
 RUN llvm-link -o pngtest-linked.bc pngtest.bc libz.bc
-RUN polytracker instrument-bc --taint --ftrace pngtest-linked.bc -o instrumented.bc
+RUN polytracker instrument-bc --cflog pngtest-linked.bc -o instrumented.bc
 RUN polytracker lower-bc instrumented.bc -t pngtest -o pngtest.instrumented

examples/Dockerfile-daedalus-nitf.demo

@@ -30,5 +30,5 @@ RUN cabal run ../../:daedalus -- compile-c++ nitf_main.ddl --out-dir=cpp_parser
 
 WORKDIR /polytracker/the_klondike/daedalus/formats/nitf/cpp_parser
 RUN polytracker build make parser && \
-    polytracker instrument-targets --taint --ftrace parser --ignore-lists gmp ssl libz && \
+    polytracker instrument-targets --cflog parser --ignore-lists gmp ssl libz && \
     mv parser.instrumented parser-track

examples/Dockerfile-daedalus-pdf.demo

@@ -4,11 +4,12 @@ RUN mkdir -p /polytracker/the_klondike
 
 WORKDIR /polytracker/the_klondike
 
-ENV DEBIAN_FRONTEND=noninteractive 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get -y upgrade && apt-get install -y git pkg-config
 
 RUN git clone --recursive https://github.com/GaloisInc/daedalus.git
 WORKDIR /polytracker/the_klondike/daedalus
+RUN git checkout 46a2502a8c68b17534079356a71af90e87801fdc
 
 FROM trailofbits/polytracker:latest
 LABEL org.opencontainers.image.authors="evan.sultanik@trailofbits.com"
@@ -30,5 +31,5 @@ WORKDIR /polytracker/the_klondike/daedalus/formats/pdf/new/c++
 
 RUN polytracker build cmake -S . -B build
 RUN polytracker build cmake --build build --target parser-test -j$(nproc)
-RUN polytracker instrument-targets --taint --ftrace parser-test --ignore-lists gmp ssl libz
+RUN polytracker instrument-targets --cflog parser-test --ignore-lists gmp ssl libz
 RUN mv parser-test.instrumented parser-test-track

examples/Dockerfile-ffmpeg.demo

@@ -27,7 +27,7 @@ RUN ../configure --disable-everything \
         --disable-asm
 
 RUN polytracker build make -j$((`nproc`+1))
-RUN polytracker instrument-targets --taint --ftrace ffmpeg_g --ignore-lists libz
+RUN polytracker instrument-targets --cflog ffmpeg_g --ignore-lists libz
 RUN mv ffmpeg_g.instrumented ffmpeg_track
 
 # Use the following command to transcode a `.mov` h264/aac file to an `.avi` raw/aac file

examples/Dockerfile-file.demo

@@ -16,5 +16,5 @@ RUN git fetch --tags && \
 RUN autoreconf -fiv
 RUN ./configure --prefix=/polytracker/the_klondike/bin/ --disable-shared
 RUN polytracker build make -j$((`nproc`+1)) install
-RUN polytracker instrument-targets --taint --ftrace file --ignore-lists libz
+RUN polytracker instrument-targets --cflog file --ignore-lists libz
 RUN mv file.instrumented file_track

examples/Dockerfile-jq.demo

@@ -11,4 +11,4 @@ WORKDIR /polytracker/the_klondike/jq
 RUN autoreconf -fi
 RUN ./configure --with-oniguruma=builtin CC=clang
 RUN polytracker build make -j$((`nproc`+1))
-RUN polytracker instrument-targets --taint --ftrace jq
+RUN polytracker instrument-targets --cflog jq

examples/Dockerfile-libjpeg.demo

@@ -18,6 +18,6 @@ WORKDIR /polytracker/the_klondike/jpeg-9e/build
 RUN ../configure LDFLAGS="-static"
 # Build and instrument
 RUN polytracker build make -j$((`nproc`+1))
-RUN polytracker instrument-targets --taint --ftrace djpeg
+RUN polytracker instrument-targets --cflog djpeg
 # Create `djpeg_track`
 RUN mv djpeg.instrumented djpeg_track

examples/Dockerfile-listgen.demo

@@ -11,7 +11,7 @@ RUN apt update
 #Update pkg-config/util-linux (needed for FontConfig)
 RUN apt update
 RUN apt install pkg-config uuid-dev gperf libtool \
-	gettext autopoint autoconf -y
+	gettext autopoint autoconf wget -y
 
 RUN apt-get install python3-dev
 
@@ -38,7 +38,7 @@ RUN make -j5 install
 WORKDIR /polytracker/the_klondike
 
 #zlib
-RUN wget https://www.zlib.net/zlib-1.2.11.tar.gz
+RUN wget https://www.zlib.net/fossils/zlib-1.2.11.tar.gz
 RUN tar -xzvf zlib-1.2.11.tar.gz
 WORKDIR zlib-1.2.11
 RUN ./configure --prefix=/usr  && make -j$(nproc) test && make -j$(nproc) install

examples/Dockerfile-mupdf.demo

@@ -4,7 +4,7 @@ RUN mkdir -p /polytracker/the_klondike
 
 WORKDIR /polytracker/the_klondike
 
-ENV DEBIAN_FRONTEND=noninteractive 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get -y upgrade && apt-get install -y git pkg-config
 
 RUN git clone --recursive --branch 1.20.0 git://git.ghostscript.com/mupdf.git
@@ -22,8 +22,8 @@ WORKDIR /polytracker/the_klondike/mupdf
 RUN git checkout d00de0e96a4a5ec90ffc30837d40cd624a6a89e0
 # Instrument mutool
 RUN polytracker build make -j$((`nproc`+1)) HAVE_X11=no HAVE_GLUT=no prefix=/usr/local build=release install
-RUN polytracker instrument-targets --taint --ftrace mutool
+RUN polytracker instrument-targets --cflog mutool
 RUN mv mutool.instrumented mutool_track
 # Note, the /workdir directory is intended to be mounted at runtime
-VOLUME ["/workdir"]
-WORKDIR /workdir
+# VOLUME ["/workdir"]
+# WORKDIR /workdir

examples/Dockerfile-nitro-nitf.demo

@@ -18,9 +18,6 @@ RUN polytracker build cmake .. \
 
 RUN polytracker build cmake --build . -j$((`nproc`+1)) --target show_nitf++
 
-RUN polytracker instrument-targets \
-	--taint \
-	--ftrace \
-	show_nitf++
+RUN polytracker instrument-targets --cflog show_nitf++
 
 RUN mv show_nitf++.instrumented nitro_track

examples/Dockerfile-openjpeg.demo

@@ -20,5 +20,5 @@ RUN polytracker extract-bc bin/opj_decompress -o opj_decompress.bc
 RUN polytracker extract-bc bin/libopenjp2.a -o libopenjp2.a.bc
 RUN llvm-link -only-needed opj_decompress.bc libopenjp2.a.bc -o exec.bc
 RUN polytracker opt-bc exec.bc -o exec.bc
-RUN polytracker instrument-bc --taint --ftrace exec.bc -o exec.bc -o exec.instrumented.bc
+RUN polytracker instrument-bc --cflog exec.bc -o exec.bc -o exec.instrumented.bc
 RUN polytracker lower-bc exec.instrumented.bc -t opj_decompress -o opj_decompress_track

examples/Dockerfile-pdfium.demo

@@ -9,6 +9,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata  \
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update     \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      curl                                            \
       git                                             \
       pkg-config                                      \
       sudo                                            \