Replace Boost.python with pybind11

.github/ci-hpc-config.yml

@@ -30,12 +30,12 @@ gnu-14.2.0:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/GNU/14.2
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/GNU/14.2/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/GNU/14.2/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.11-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -54,15 +54,15 @@ gnu-12.2.0:
     cmake_options:
       - >-
         -DCMAKE_VERBOSE_MAKEFILE=ON
-        -DENABLE_ALL_TESTS=ON 
-        -DBoost_ROOT=/usr/local/apps/boost/1.81.0/GNU/12.2 
-        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/GNU/12.2/include 
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/GNU/12.2/lib/libboost_python310.so 
-        -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01 
-        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3 
-        -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10 
-        -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so 
+        -DENABLE_ALL_TESTS=ON
+        -DBoost_ROOT=/usr/local/apps/boost/1.81.0/GNU/12.2
+        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/GNU/12.2/include
+        -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
+        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
+        -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
+        -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -82,14 +82,14 @@ gnu-8.5.0:
       - >-
         -DCMAKE_VERBOSE_MAKEFILE=ON
         -DENABLE_ALL_TESTS=ON
-        -DBoost_ROOT=/usr/local/apps/boost/1.84.0/GNU/8.5 
-        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.84.0/GNU/8.5/include 
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.84.0/GNU/8.5/lib/libboost_python311.so 
+        -DBoost_ROOT=/usr/local/apps/boost/1.84.0/GNU/8.5
+        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.84.0/GNU/8.5/include
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.11.8-01
-        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.11.8-01/bin/python3 
+        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.11.8-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.11.8-01/include/python3.11
-        -DPython3_LIBRARIES=/usr/local/apps/python3/3.11.8-01/lib64/libpython3.11.so 
+        -DPython3_LIBRARIES=/usr/local/apps/python3/3.11.8-01/lib64/libpython3.11.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.11.8-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.11.8-01/lib/python3.11/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -112,12 +112,12 @@ intel-2025.0.1:
         -DCMAKE_CXX_COMPILER=icpx
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/INTEL/2025.0
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/INTEL/2025.0/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/INTEL/2025.0/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.9-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -140,12 +140,12 @@ intel-2021.4.0:
         -DCMAKE_CXX_COMPILER=icpx
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/INTEL/2021.4
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/INTEL/2021.4/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/INTEL/2021.4/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       - -L nightly -E py3_
 
@@ -166,12 +166,12 @@ nvidia-24.11:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/NVIDIA/24.11
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/NVIDIA/24.11/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/NVIDIA/24.11/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.9-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable all tests
       - --version
@@ -193,12 +193,12 @@ nvidia-22.11:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/NVIDIA/22.11
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/NVIDIA/22.11/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/NVIDIA/22.11/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options: 
       # disable all tests
       - --version
@@ -220,12 +220,12 @@ aocc-4.0.0:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/AMD/4.0
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/AMD/4.0/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/AMD/4.0/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable all tests
       - --version

cmake/Dependencies.cmake

@@ -123,6 +123,25 @@ if (ENABLE_PYTHON)
 endif()
 
 
+# =========================================================================================
+# pybind11
+# =========================================================================================
+if (ENABLE_PYTHON)
+
+  ecbuild_info( "Locating pybind11" )
+
+  find_package(pybind11 2.10.3 REQUIRED)
+
+  ecbuild_info( "pybind11 details:" )
+  ecbuild_info( " * pybind11_FOUND        : ${pybind11_FOUND}" )
+  ecbuild_info( " * pybind11_INCLUDE_DIRS : ${pybind11_INCLUDE_DIRS}" )
+  ecbuild_info( " * pybind11_VERSION      : ${pybind11_VERSION}" )
+  ecbuild_info( " * pybind11_VERSION_MAJOR: ${pybind11_VERSION_MAJOR}" )
+  ecbuild_info( " * pybind11_VERSION_MINOR: ${pybind11_VERSION_MINOR}" )
+  ecbuild_info( " * pybind11_VERSION_PATCH: ${pybind11_VERSION_PATCH}" )
+
+endif()
+
 # =========================================================================================
 # Boost
 # =========================================================================================
@@ -174,21 +193,6 @@ if ( Boost_MINOR_VERSION GREATER_EQUAL 86 )
   list(APPEND _boost_needed_libs process)
 endif()
 
-if (ENABLE_PYTHON)
-  # The following is used to find Boost.python library, as the library name changes with python version
-  if ( Boost_MINOR_VERSION GREATER 66 )
-    # cmake 3.15
-    # see: https://gitlab.kitware.com/cmake/cmake/issues/19656
-    # INTERFACE_LIBRARY targets may only have whitelisted properties.
-    set(_python_base_version "${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}")
-  else()
-    set(_python_base_version "${Python3_VERSION_MAJOR}")
-  endif()
-  set(ECFLOW_BOOST_PYTHON_COMPONENT "python${_python_base_version}")
-
-  list(APPEND _boost_needed_libs ${ECFLOW_BOOST_PYTHON_COMPONENT})
-endif()
-
 if(HAVE_TESTS) # HAVE_TESTS is defined if ecbuild ENABLE_TESTS is set, (by default this is set)
   list(APPEND _boost_needed_libs unit_test_framework test_exec_monitor )
 endif()

libs/attribute/src/ecflow/attribute/CronAttr.hpp

@@ -77,14 +77,10 @@ class CronAttr {
     const TimeSeries& time() const { return timeSeries_; }
     const TimeSeries& time_series() const { return timeSeries_; }
     bool last_day_of_month() const { return last_day_of_month_; }
-    std::vector<int>::const_iterator week_days_begin() const { return weekDays_.begin(); }
-    std::vector<int>::const_iterator week_days_end() const { return weekDays_.end(); }
-    std::vector<int>::const_iterator last_week_days_of_month_begin() const { return last_week_days_of_month_.begin(); }
-    std::vector<int>::const_iterator last_week_days_end_of_month_end() const { return last_week_days_of_month_.end(); }
-    std::vector<int>::const_iterator days_of_month_begin() const { return daysOfMonth_.begin(); }
-    std::vector<int>::const_iterator days_of_month_end() const { return daysOfMonth_.end(); }
-    std::vector<int>::const_iterator months_begin() const { return months_.begin(); }
-    std::vector<int>::const_iterator months_end() const { return months_.end(); }
+    const std::vector<int>& week_days() const { return weekDays_; }
+    const std::vector<int>& last_week_days_of_month() const { return last_week_days_of_month_; }
+    const std::vector<int>& days_of_month() const { return daysOfMonth_; }
+    const std::vector<int>& months() const { return months_; }
 
     std::string name() const; // for display/gui
     std::string toString() const;

libs/client/src/ecflow/client/ClientInvoker.hpp

@@ -458,6 +458,8 @@ class ClientInvoker {
      */
     bool is_not_retrying(const ClientToServerCmd& cmd) const;
 
+    auto& changed_node_paths() { return server_reply_.changed_nodes(); }
+
 private:
     /**
      * @return 1 when command is selected; 0 if no command is selected (e.g. --help)

libs/pyext/CMakeLists.txt

@@ -57,6 +57,7 @@ set(srcs
   ../src/ecflow/python/ClientDoc.hpp
   ../src/ecflow/python/DefsDoc.hpp
   ../src/ecflow/python/Edit.hpp
+  ../src/ecflow/python/ExportCollections.hpp
   ../src/ecflow/python/GlossaryDoc.hpp
   ../src/ecflow/python/NodeAttrDoc.hpp
   ../src/ecflow/python/NodeUtil.hpp
@@ -69,6 +70,7 @@ set(srcs
   ../src/ecflow/python/EcfExt.cpp
   ../src/ecflow/python/Edit.cpp
   ../src/ecflow/python/ExportClient.cpp
+  ../src/ecflow/python/ExportCollections.cpp
   ../src/ecflow/python/ExportCore.cpp
   ../src/ecflow/python/ExportDefs.cpp
   ../src/ecflow/python/ExportNode.cpp
@@ -129,6 +131,6 @@ set(s_tests
   s_TestSslSetup
 )
 
-if (Python3_FOUND AND (Boost_PYTHON3_FOUND OR Boost_PYTHON${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}_FOUND))
+if (Python3_FOUND AND pybind11_FOUND)
   add_subdirectory( python3 )
 endif()

libs/pyext/python3/CMakeLists.txt

@@ -32,7 +32,7 @@ ecbuild_add_library(
     ecflow_all
     libsimulator
     Python3::Module
-    Boost::${ECFLOW_BOOST_PYTHON_COMPONENT}
+    pybind11::headers
     $<$<BOOL:${OPENSSL_FOUND}>:OpenSSL::SSL>
   CXXFLAGS
     $<$<CXX_COMPILER_ID:Clang>:-Wno-macro-redefined>

libs/pyext/src/ecflow/python/ClientDoc.cpp

@@ -40,13 +40,13 @@ const char* ClientDoc::class_client() {
            "* ECF_DENIED <any> : Provides a way for child to exit with an error, if server denies connection. Avoids "
            "24hr wait. Note: when you have hundreds of tasks, using this approach requires a lot of manual "
            "intervention to determine job status\n"
-           "* NO_ECF <any> : If set exit's immediately with success. Used to test jobs without communicating with "
+           "* NO_ECF <any> : If set exits immediately with success. Used to test jobs without communicating with "
            "server\n\n"
            "The following environment variables are used by the python interface and child commands\n\n"
            "* ECF_HOST  <string>   : The host name of the main server. defaults to 'localhost'\n"
            "* ECF_PORT  <int>      : The TCP/IP port to call on the server. Must be unique to a server\n\n"
            "The ECF_HOST and ECF_PORT can be overridden by using the Constructor or set_host_port() member function.\n"
-           "By default the Client interface will throw exceptions for error's.\n"
+           "By default the Client interface will throw exceptions for errors.\n"
            "\nUsage:\n\n"
            ".. code-block:: python\n\n"
            "   try:\n"
@@ -310,7 +310,7 @@ const char* ClientDoc::load_defs() {
            ".. code-block:: python\n\n"
            "   defs_file = 'Hello.def' \n"
            "   defs = Defs()\n"
-           "   suite = def.add_suite('s1')\n"
+           "   suite = defs.add_suite('s1')\n"
            "   family = suite.add_family('f1')\n"
            "   for i in [ '_1', '_2', '_3' ]:\n"
            "      family.add_task( 't' + i )\n"
@@ -329,7 +329,7 @@ const char* ClientDoc::load() {
            "      Defs defs           : A in memory definition\n"
            "      [(bool)force=False] : for true overwrite suite of same name\n"
            "   )\n\n"
-           "If force is not used and `suite`_ already exists in the server, then a error is thrown.\n"
+           "If force is not used and `suite`_ already exists in the server, then an error is thrown.\n"
            "\nUsage:\n\n"
            ".. code-block:: python\n\n"
            "   defs = Defs()\n"

libs/pyext/src/ecflow/python/DefsDoc.cpp

@@ -143,7 +143,7 @@ const char* DefsDoc::expression_doc() {
 }
 
 const char* DefsDoc::add_trigger_doc() {
-    return "Add a `trigger`_ or `complete expression`_.Also see :py:class:`ecflow.Trigger`\n\n"
+    return "Add a `trigger`_ or `complete expression`_. Also see :py:class:`ecflow.Trigger`\n\n"
            "This defines a dependency for a `node`_.\n"
            "There can only be one `trigger`_ or `complete expression`_ dependency per node.\n"
            "A `node`_ with a trigger can only be activated when the trigger has expired.\n"
@@ -279,7 +279,7 @@ const char* DefsDoc::suite_doc() {
            "There are several ways of adding a suite, see example below and :py:class:`ecflow.Defs.add_suite`\n"
            "\nConstructor::\n\n"
            "  Suite(name, Nodes | attributes)\n"
-           "      string name : The Suite name. name must consist of alpha numeric characters or\n"
+           "      string name : The Suite name. name must consist of alphanumeric characters or\n"
            "                    underscore or dot. The first character cannot be a dot, as this\n"
            "                    will interfere with trigger expressions. Case is significant\n"
            "      Nodes | Attributes:(optional)\n"
@@ -299,15 +299,15 @@ const char* DefsDoc::suite_doc() {
 }
 
 const char* DefsDoc::family_doc() {
-    return "Create a `family`_ `node`_.A Family node lives inside a `suite`_ or another `family`_\n\n"
+    return "Create a `family`_ `node`_. A Family node lives inside a `suite`_ or another `family`_\n\n"
            "A family is used to collect `task`_\\ s together or to group other families.\n"
            "Typically you place tasks that are related to each other inside the same family\n"
            "analogous to the way you create directories to contain related files.\n"
            "There are two ways of adding a family, see example below.\n"
            "\nConstructor::\n\n"
            "  Family(name, Nodes | Attributes)\n"
-           "      string name : The Family name. name must consist of alpha numeric characters or\n"
-           "                    underscore or dot. The first character cannot be dot, as this\n"
+           "      string name : The Family name. name must consist of alphanumeric characters or\n"
+           "                    underscore or dot. The first character cannot be a dot, as this\n"
            "                    will interfere with trigger expressions. Case is significant\n"
            "      Nodes | Attributes: (optional)\n"
            "\nException:\n\n"
@@ -336,7 +336,7 @@ const char* DefsDoc::task_doc() {
            "There are several ways of adding a task, see examples below\n"
            "\nConstructor::\n\n"
            "  Task(name, Attributes)\n"
-           "     string name : The Task name.Name must consist of alpha numeric characters or\n"
+           "     string name : The Task name. Name must consist of alphanumeric characters or\n"
            "                   underscore or dot. First character cannot be a dot.\n"
            "                   Case is significant\n"
            "     attributes: optional, i.e like Meter, Event, Trigger etc\n"
@@ -407,8 +407,8 @@ const char* DefsDoc::add_extern_doc() {
            "Automatic Method:\n"
            "  This will scan all trigger and complete expressions, looking for paths and variables\n"
            "  that have not been defined. The added benefit of this approach is that duplicates will not\n"
-           "  be added. It is the user's responsibility to check that extern's are eventually defined\n"
-           "  otherwise trigger expression will not evaluate correctly\n\n"
+           "  be added. It is the user's responsibility to check that externs are eventually defined\n"
+           "  otherwise trigger expressions will not evaluate correctly\n\n"
            ".. code-block:: python\n\n"
            "  void auto_add_externs(bool remove_existing_externs_first )\n"
            "\nUsage:\n\n"
@@ -450,7 +450,7 @@ const char* DefsDoc::add_family_doc() {
 const char* DefsDoc::add_task_doc() {
     return "Add a `task`_. See :py:class:`ecflow.Task`\n\n"
            "Multiple Tasks can be added. However Task names must be unique,\n"
-           "for a given parent. Task can be added to Familiy's or Suites.\n"
+           "for a given parent. Task can be added to Families or Suites.\n"
            "\nException:\n\n"
            "- Throws RuntimeError if a duplicate is added\n"
            "\nUsage:\n\n"
@@ -469,7 +469,7 @@ const char* DefsDoc::add_definition_doc() {
            "If a definition is read in from disk, it will by default, check the `trigger`_ expressions.\n"
            "If however the definition is created in python, then checking should be done explicitly:\n\n"
            "   Defs(string)\n"
-           "      string - The Defs class take one argument which represents the file name\n"
+           "      string - The Defs class takes one argument which represents the file name\n"
            "   Defs(Suite | Edit )\n"
            "      :py:class:`ecflow.Suite`- One or more suites\n\n"
            "      :py:class:`ecflow.Edit` - specifies user defined server variables\n"