Clean up parsing of multi-member PDFs

.github/workflows/run-capi.yml

@@ -1,4 +1,4 @@
-name: Run the C/C++-API tests 📦
+name: Run the C/C++/Fortran-API tests 📦
 
 on: [push]
 

docs/examples/c-oop.md

@@ -1,7 +1,7 @@
 # C++ OOP API Example
 
 This example briefly demonstrates how to use the `NeoPDF` C++ OOP API to load and evaluate parton
-distributions.
+distributions. More examples can be found in [neopdf_capi/tests](https://github.com/Radonirinaunimi/neopdf/tree/master/neopdf_capi/tests).
 
 ## Prerequisites
 
@@ -26,7 +26,6 @@ to LHAPDF.
 ```cpp linenums="1"
 #include <LHAPDF/PDF.h>
 #include <LHAPDF/GridPDF.h>
-#include <neopdf_capi.h>
 #include <NeoPDF.hpp>
 #include <cassert>
 #include <cmath>
@@ -230,13 +229,54 @@ void test_all_pdf_members() {
     std::cout << "Relative Std Dev: " << std_dev / mean << "\n";
 }
 
-void test_raw_load_all() {
-    std::cout << "=== Test raw neopdf_pdf_load_all ===\n";
-    NeoPDFMembers raw_pdfs = neopdf_pdf_load_all("NNPDF40_nnlo_as_01180");
-    std::cout << "Loaded " << raw_pdfs.size << " PDF members (raw call)\n";
-    neopdf_pdf_array_free(raw_pdfs);
-}
+void test_lazy_loading() {
+    std::cout << "=== Test NeoPDFLazy class (lazy loading) ===\n";
+
+    // Disable LHAPDF banners to guarantee deterministic output
+    LHAPDF::setVerbosity(0);
+
+    std::string pdfname = "NNPDF40_nnlo_as_01180.neopdf.lz4";
+    NeoPDFLazy lazy_pdfs(pdfname);
+
+    std::cout << "Initialized lazy loader for " << pdfname << "\n";
+
+    // Test case: evaluate a simple point across all members
+    int pid = 1;
+    double x = 1e-9;
+    double q2 = 1.65 * 1.65;
 
+    std::cout << std::right
+        << std::setw(8) << "Member"
+        << std::setw(15) << "LHAPDF"
+        << std::setw(15) << "NeoPDF"
+        << std::setw(15) << "Rel. Diff." << "\n";
+    std::cout << std::string(53, '-') << "\n";
+
+    // Evaluate the same point across all PDF members
+    std::vector<double> results;
+    int member_idx = 0;
+    while (auto neo_pdf = lazy_pdfs.next()) {
+        const LHAPDF::PDF* basepdf = LHAPDF::mkPDF("NNPDF40_nnlo_as_01180", member_idx);
+        const LHAPDF::GridPDF& lha_pdf = *dynamic_cast<const LHAPDF::GridPDF*>(basepdf);
+
+        double expected = lha_pdf.xfxQ2(pid, x, q2);
+        double result = neo_pdf->xfxQ2(pid, x, q2);
+
+        double reldif = std::abs(result - expected) / expected;
+        assert(std::abs(result - expected) < TOLERANCE);
+        results.push_back(result);
+
+        std::cout << std::right
+            << std::setw(8) << member_idx
+            << std::scientific << std::setprecision(8)
+            << std::setw(15) << expected
+            << std::setw(15) << result
+            << std::setw(15) << reldif << "\n";
+        member_idx++;
+    }
+
+    std::cout << "\nSuccessfully iterated through all members lazily.\n";
+}
 
 int main() {
     // Test the computation of the PDF interpolations
@@ -248,6 +288,9 @@ int main() {
     // Test the PDF interpolations by loading all the members
     test_all_pdf_members();
 
+    // Test the lazy loading of PDF members
+    test_lazy_loading();
+
     return EXIT_SUCCESS;
 }
 ```

docs/examples/c.md

@@ -1,7 +1,7 @@
 # C API Example
 
 This example demonstrates how to use the `NeoPDF` C API to load and evaluate parton
-distributions.
+distributions. More examples can be found in [neopdf_capi/tests](https://github.com/Radonirinaunimi/neopdf/tree/master/neopdf_capi/tests).
 
 ## Prerequisites
 
@@ -177,6 +177,69 @@ void test_all_pdf_members() {
     neopdf_pdf_array_free(neo_pdfs);
 }
 
+void test_lazy_loading() {
+    std::cout << "=== Test Lazy Loading of PDF Members ===\n";
+
+    // Disable LHAPDF banners to guarantee deterministic output
+    LHAPDF::setVerbosity(0);
+
+    std::string pdfname = "NNPDF40_nnlo_as_01180";
+    std::string neopdf_name = pdfname + ".neopdf.lz4";
+    NeoPDFLazyIterator* lazy_iter = neopdf_pdf_load_lazy(neopdf_name.c_str());
+
+    if (!lazy_iter) {
+        std::cerr << "Failed to load lazy iterator for " << pdfname << std::endl;
+        return;
+    }
+
+    std::cout << "Successfully loaded lazy iterator for " << pdfname << "\n";
+
+    // Test case: evaluate a simple point across all members
+    int pid = 1;
+    double x = 1e-9;
+    double q2 = 1.65 * 1.65;
+
+    std::cout << "\nEvaluating xfxQ2 for pid=" << pid
+              << ", x=" << std::scientific << x
+              << ", Q2=" << q2 << " across all members (lazily):\n";
+
+    std::cout << std::right
+        << std::setw(8) << "Member"
+        << std::setw(15) << "LHAPDF"
+        << std::setw(15) << "NeoPDF"
+        << std::setw(15) << "Rel. Diff." << "\n";
+    std::cout << std::string(53, '-') << "\n";
+
+    // Evaluate the same point across all PDF members
+    std::vector<double> results;
+    int member_idx = 0;
+    while (NeoPDFWrapper* pdf = neopdf_lazy_iterator_next(lazy_iter)) {
+        auto lha_pdf = std::unique_ptr<LHAPDF::PDF>(LHAPDF::mkPDF(pdfname, member_idx));
+
+        double expected = lha_pdf->xfxQ2(pid, x, q2);
+        double result = neopdf_pdf_xfxq2(pdf, pid, x, q2);
+
+        double reldif = std::abs(result - expected) / expected;
+        assert(std::abs(result - expected) < TOLERANCE);
+        results.push_back(result);
+
+        std::cout << std::right
+            << std::setw(8) << member_idx
+            << std::scientific << std::setprecision(8)
+            << std::setw(15) << expected
+            << std::setw(15) << result
+            << std::setw(15) << reldif << "\n";
+
+        neopdf_pdf_free(pdf); // Free the individual PDF member
+        member_idx++;
+    }
+
+    // Free the lazy iterator
+    neopdf_lazy_iterator_free(lazy_iter);
+
+    std::cout << "\nSuccessfully iterated through all members lazily.\n";
+}
+
 
 int main() {
     // Test loading single PDF member
@@ -185,6 +248,9 @@ int main() {
     // Test loading all the PDF members
     test_all_pdf_members();
 
+    // Test lazy loading of PDF members
+    test_lazy_loading();
+
     return EXIT_SUCCESS;
 }
 ```

neopdf/src/parser.rs

@@ -4,12 +4,12 @@
 //! set formats, including subgrid data extraction and metadata reading.
 use serde::{Deserialize, Serialize};
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use super::gridpdf::GridArray;
 use super::manage::{ManageData, PdfSetFormat};
 use super::metadata::MetaData;
-use super::writer::GridArrayReader;
+use super::writer::{GridArrayReader, LazyGridArrayIterator};
 
 /// Represents the data for a single subgrid within a PDF data file.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -200,24 +200,34 @@ impl LhapdfSet {
 pub struct NeopdfSet {
     pub info: MetaData,
     grid_reader: GridArrayReader,
+    setpath: PathBuf,
 }
 
 impl NeopdfSet {
     /// TODO
     pub fn new(pdf_name: &str) -> Self {
         let manager = ManageData::new(pdf_name, PdfSetFormat::Neopdf);
         let neopdf_setpath = manager.set_path();
-        let grid_reader = GridArrayReader::from_file(neopdf_setpath).unwrap();
-        let info = grid_reader.metadata().as_ref().clone();
+        let grid_readers = GridArrayReader::from_file(neopdf_setpath).unwrap();
+        let metadata_info = grid_readers.metadata().as_ref().clone();
 
-        Self { info, grid_reader }
+        Self {
+            info: metadata_info,
+            grid_reader: grid_readers,
+            setpath: neopdf_setpath.to_path_buf(),
+        }
     }
 
     /// TODO
     pub fn member(&self, member: usize) -> (MetaData, GridArray) {
         let load_grid = self.grid_reader.load_grid(member).unwrap();
         (self.info.clone(), load_grid.grid)
     }
+
+    /// TODO
+    pub fn into_lazy_iterators(&self) -> LazyGridArrayIterator {
+        LazyGridArrayIterator::from_file(&self.setpath).unwrap()
+    }
 }
 
 #[cfg(test)]

neopdf/src/pdf.rs

@@ -73,6 +73,26 @@ fn pdfset_loader<T: PdfSet>(set: T, member: usize) -> PDF {
     }
 }
 
+/// Loads all PDF members from a generic PDF set backend in sequential.
+///
+/// # Arguments
+///
+/// * `set` - The PDF set backend implementing [`PdfSet`].
+///
+/// # Returns
+///
+/// A vector of [`PDF`] instances, one for each member in the set.
+fn pdfsets_seq_loader<T: PdfSet + Send + Sync>(set: T) -> Vec<PDF> {
+    (0..set.num_members())
+        .map(|idx| {
+            let (info, knot_array) = set.member(idx);
+            PDF {
+                grid_pdf: GridPDF::new(info, knot_array),
+            }
+        })
+        .collect()
+}
+
 /// Loads all PDF members from a generic PDF set backend in parallel.
 ///
 /// # Arguments
@@ -82,7 +102,7 @@ fn pdfset_loader<T: PdfSet>(set: T, member: usize) -> PDF {
 /// # Returns
 ///
 /// A vector of [`PDF`] instances, one for each member in the set.
-fn pdfsets_loader<T: PdfSet + Send + Sync>(set: T) -> Vec<PDF> {
+fn pdfsets_par_loader<T: PdfSet + Send + Sync>(set: T) -> Vec<PDF> {
     (0..set.num_members())
         .into_par_iter()
         .map(|idx| {
@@ -125,7 +145,7 @@ impl PDF {
         }
     }
 
-    /// Loads all members of a PDF set.
+    /// Loads all members of a PDF set in parallel.
     ///
     /// This function reads the `.info` file and all `.dat` member files
     /// to construct a `Vec<PDF>`, with each `PDF` instance representing a member
@@ -140,12 +160,67 @@ impl PDF {
     /// A `Vec<PDF>` where each element is a `PDF` instance for a member of the set.
     pub fn load_pdfs(pdf_name: &str) -> Vec<PDF> {
         if pdf_name.ends_with(".neopdf.lz4") {
-            pdfsets_loader(NeopdfSet::new(pdf_name))
+            pdfsets_par_loader(NeopdfSet::new(pdf_name))
         } else {
-            pdfsets_loader(LhapdfSet::new(pdf_name))
+            pdfsets_par_loader(LhapdfSet::new(pdf_name))
         }
     }
 
+    /// Loads all members of a PDF set in sequential.
+    ///
+    /// This function reads the `.info` file and all `.dat` member files
+    /// to construct a `Vec<PDF>`, with each `PDF` instance representing a member
+    /// of the set. The loading is performed in parallel.
+    ///
+    /// # Arguments
+    ///
+    /// * `pdf_name` - The name of the PDF set.
+    ///
+    /// # Returns
+    ///
+    /// A `Vec<PDF>` where each element is a `PDF` instance for a member of the set.
+    pub fn load_pdfs_seq(pdf_name: &str) -> Vec<PDF> {
+        if pdf_name.ends_with(".neopdf.lz4") {
+            pdfsets_seq_loader(NeopdfSet::new(pdf_name))
+        } else {
+            pdfsets_seq_loader(LhapdfSet::new(pdf_name))
+        }
+    }
+
+    /// Creates an iterator that loads PDF members lazily.
+    ///
+    /// This function is suitable for `.neopdf.lz4` files, which support lazy loading.
+    /// It returns an iterator that yields `PDF` instances on demand, which is useful
+    /// for reducing memory consumption when working with large PDF sets.
+    ///
+    /// # Arguments
+    ///
+    /// * `pdf_name` - The name of the PDF set (must end with `.neopdf.lz4`).
+    ///
+    /// # Returns
+    ///
+    /// An iterator over `Result<PDF, Box<dyn std::error::Error>>`.
+    pub fn load_pdfs_lazy(
+        pdf_name: &str,
+    ) -> impl Iterator<Item = Result<PDF, Box<dyn std::error::Error>>> {
+        assert!(
+            pdf_name.ends_with(".neopdf.lz4"),
+            "Lazy loading is only supported for .neopdf.lz4 files"
+        );
+
+        let iter_lazy = NeopdfSet::new(pdf_name).into_lazy_iterators();
+
+        iter_lazy.map(|grid_array_with_metadata_result| {
+            grid_array_with_metadata_result.map(|grid_array_with_metadata| {
+                let info = (*grid_array_with_metadata.metadata).clone();
+                let knot_array = grid_array_with_metadata.grid;
+                PDF {
+                    grid_pdf: GridPDF::new(info, knot_array),
+                }
+            })
+        })
+    }
+
     /// Clip the negative values for the `PDF` object.
     ///
     /// # Arguments

neopdf/src/writer.rs

@@ -10,8 +10,8 @@
 //! - Compression and decompression of multiple [`GridArray`]s with shared metadata using LZ4 and bincode
 //!   serialization.
 //! - Random access to individual grid members without loading the entire collection into memory.
-//! - Lazy iteration over grid members for memory-efficient processing of large sets.
 //! - Extraction of metadata without full decompression.
+//! - Lazy iteration over grid members for memory-efficient processing of large sets.
 //!
 //! # Key Types
 //!
@@ -366,6 +366,11 @@ impl LazyGridArrayIterator {
 
         let count: u64 = bincode::deserialize_from(&mut cursor)?;
 
+        // Read and skip the offset table
+        let offset_table_size: u64 = bincode::deserialize_from(&mut cursor)?;
+        let mut offset_table_bytes = vec![0u8; offset_table_size as usize];
+        cursor.read_exact(&mut offset_table_bytes)?;
+
         Ok(Self {
             cursor,
             remaining: count,

neopdf/tests/pdf.rs

@@ -236,6 +236,13 @@ pub fn test_xfxq2s() {
     }
 }
 
+#[test]
+pub fn test_multi_members_loader() {
+    let pdfs = PDF::load_pdfs("NNPDF40_nnlo_as_01180");
+
+    assert!(pdfs.len() == 101);
+}
+
 #[test]
 pub fn test_boundary_extraction() {
     let pdf = PDF::load("NNPDF40_nnlo_as_01180", 0);