Improve performance of the client

.github/workflows/build-docs.yml

@@ -0,0 +1,40 @@
+name: Build documentation
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  test:
+    name: Build docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          pip install cmake pybind11 numpy tox lz4
+
+      - name: Build docs
+        run: |
+          python setup.py build_ext --inplace
+          cp build/lib*/wobbegong/lib_wobbegong* src/wobbegong/
+          tox -e docs
+          touch ./docs/_build/html/.nojekyll
+
+      - name: GH Pages Deployment
+        if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: gh-pages # The branch the action should deploy to.
+          folder: ./docs/_build/html
+          clean: true # Automatically remove deleted files from the deploy branch

.github/workflows/publish-pypi.yml

@@ -2,51 +2,64 @@ name: Publish to PyPI
 
 on:
   push:
-    tags: "*"
+    tags:
+      - "*"
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      repository-projects: write
-      contents: write
-      pages: write
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # macos-13 is an intel runner, higher macos's are apple silicon
+        # At some point, maybe get this to work on windows-latest
+        os: [ubuntu-latest, macos-latest]
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.22.0
+        env:
+          CIBW_ARCHS_LINUX: x86_64
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
+          CIBW_SKIP: pp*
 
-      - name: Set up Python 3.12
-        uses: actions/setup-python@v5
+      - uses: actions/upload-artifact@v4
         with:
-          python-version: 3.12
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
 
-      - name: Test with tox
-        run: |
-          tox
+      - name: Build sdist
+        run: pipx run build --sdist
 
-      - name: Build Project and Publish
-        run: |
-          python -m tox -e clean,build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
 
       # This uses the trusted publisher workflow so no token is required.
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
-
-      - name: Build docs
-        run: |
-          tox -e docs
-
-      - run: touch ./docs/_build/html/.nojekyll
-
-      - name: GH Pages Deployment
-        uses: JamesIves/github-pages-deploy-action@v4
-        with:
-          branch: gh-pages # The branch the action should deploy to.
-          folder: ./docs/_build/html
-          clean: true # Automatically remove deleted files from the deploy branch

.github/workflows/run-tests.yml

@@ -48,6 +48,9 @@ jobs:
           python -m pip install --upgrade pip
           pip install tox coverage
 
+      - name: Get latest CMake
+        uses: lukka/get-cmake@latest
+
       - name: Run tests
         run: >-
           pipx run --python '${{ steps.setup-python.outputs.python-path }}'

lib/CMakeLists.txt

@@ -0,0 +1,39 @@
+cmake_minimum_required(VERSION 3.24)
+
+project(wobbegong
+    VERSION 1.0.0
+    DESCRIPTION "Building the wobbegong cpp library"
+    LANGUAGES CXX)
+
+include(FetchContent)
+
+FetchContent_Declare(
+    lz4
+    GIT_REPOSITORY https://github.com/lz4/lz4.git
+    GIT_TAG        v1.10.0
+    SOURCE_SUBDIR  build/cmake
+)
+
+set(LZ4_BUILD_LEGACY_LZ4C OFF CACHE BOOL "" FORCE)
+set(LZ4_BUILD_CLI OFF CACHE BOOL "" FORCE)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+FetchContent_MakeAvailable(lz4)
+
+find_package(pybind11 CONFIG)
+
+pybind11_add_module(wobbegong
+    src/reader.cpp
+)
+
+set_property(TARGET wobbegong PROPERTY CXX_STANDARD 17)
+
+find_package(ZLIB REQUIRED)
+
+target_link_libraries(wobbegong PRIVATE lz4_static pybind11::pybind11 ZLIB::ZLIB)
+
+# Ensure the output name matches what Python expects
+set_target_properties(wobbegong PROPERTIES
+    OUTPUT_NAME lib_wobbegong
+    PREFIX ""
+)

lib/src/reader.cpp

@@ -0,0 +1,113 @@
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+#include <pybind11/stl.h>
+#include <vector>
+#include <string>
+#include <thread>
+#include <future>
+#include <algorithm>
+#include <numeric>
+#include <iostream>
+#include <cstring>
+
+#include <zlib.h>
+#include <lz4.h> 
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace py = pybind11;
+
+class MMapFile {
+public:
+    const char* data = nullptr;
+    size_t size = 0;
+    int fd = -1;
+
+    MMapFile(const std::string& path) {
+        fd = open(path.c_str(), O_RDONLY);
+        if (fd == -1) throw std::runtime_error("Could not open file: " + path);
+
+        struct stat sb;
+        if (fstat(fd, &sb) == -1) throw std::runtime_error("fstat failed");
+        size = sb.st_size;
+
+        if (size > 0) {
+            data = (const char*)mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+            if (data == MAP_FAILED) throw std::runtime_error("mmap failed");
+        }
+    }
+
+    ~MMapFile() {
+        if (data && size > 0) munmap((void*)data, size);
+        if (fd != -1) close(fd);
+    }
+};
+
+size_t decompress_block(const char* src, size_t src_size, std::vector<char>& dst, const std::string& algo) {
+    if (src_size == 0) return 0;
+
+    if (dst.empty()) dst.resize(std::max((size_t)1024, src_size * 4));
+
+    while (true) {
+        if (algo == "lz4") {
+            int res = LZ4_decompress_safe(src, dst.data(), static_cast<int>(src_size), static_cast<int>(dst.size()));
+            if (res >= 0) return static_cast<size_t>(res);
+            dst.resize(dst.size() * 2);
+        } else {
+            uLongf dLen = dst.size();
+            int res = uncompress(reinterpret_cast<Bytef*>(dst.data()), &dLen, reinterpret_cast<const Bytef*>(src), src_size);
+
+            if (res == Z_OK) return dLen;
+            if (res == Z_BUF_ERROR) {
+                dst.resize(dst.size() * 2);
+            } else {
+                throw std::runtime_error("Zlib decompression failed with error code: " + std::to_string(res));
+            }
+        }
+    }
+}
+
+py::bytes decompress_wrapper(py::bytes src_bytes, const std::string& compression) {
+    std::string src_str = src_bytes;
+    std::vector<char> dst;
+
+    size_t out_size;
+
+    {
+        py::gil_scoped_release release;
+        out_size = decompress_block(
+            src_str.data(),
+            src_str.size(),
+            dst,
+            compression
+        );
+    }
+
+    return py::bytes(dst.data(), out_size);
+}
+
+PYBIND11_MODULE(lib_wobbegong, m) {
+    m.doc() = "Wobbegong high-performance C++ reader/writer bindings";
+
+    m.def(
+        "decompress",
+        &decompress_wrapper,
+        py::arg("data"),
+        py::arg("compression") = "zlib",
+        R"pbdoc(
+            Decompress a compressed byte buffer.
+
+            Args:
+                data:
+                    Compressed data buffer.
+
+                compression:
+                    Compression algorithm ("zlib" or "lz4").
+
+            Returns:
+                Decompressed data.
+        )pbdoc"
+    );
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 # AVOID CHANGING REQUIRES: IT WILL BE UPDATED BY PYSCAFFOLD!
-requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5"]
+requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5", "cmake", "pybind11", "numpy", "lz4"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]