Refactor with IWYU

include/lbann/base.hpp

@@ -27,7 +27,9 @@
 #ifndef LBANN_BASE_HPP_INCLUDED
 #define LBANN_BASE_HPP_INCLUDED
 
-#include <El.hpp>
+#include "lbann/lbann_stl.hpp"
+
+#include <El.hpp> // IWYU pragma: export
 
 // Defines, among other things, DataType.
 #include "lbann_config.hpp"

include/lbann/data_readers/data_reader_cifar10.hpp

@@ -30,6 +30,8 @@
 #define LBANN_DATA_READER_CIFAR10_HPP
 
 #include "data_reader_image.hpp"
+#include "lbann/base.hpp"
+#include "lbann/lbann_stl.hpp"
 
 namespace lbann {
 

include/lbann/data_readers/data_reader_merge_features.hpp

@@ -30,8 +30,12 @@
 #define LBANN_DATA_READER_MERGE_FEATURES_HPP
 
 #include "compound_data_reader.hpp"
+#include "lbann/base.hpp"
+#include "lbann/data_readers/data_reader.hpp"
+#include "lbann/lbann_stl.hpp"
 
 namespace lbann {
+class thread_pool;
 
 /**
  * Data reader for merging multiple data readers.

include/lbann/data_readers/data_reader_merge_samples.hpp

@@ -30,6 +30,9 @@
 #define LBANN_DATA_READER_MERGE_SAMPLES_HPP
 
 #include "compound_data_reader.hpp"
+#include "lbann/base.hpp"
+#include "lbann/data_readers/data_reader.hpp"
+#include "lbann/lbann_stl.hpp"
 
 namespace lbann {
 

include/lbann/data_readers/data_reader_mesh.hpp

@@ -30,8 +30,11 @@
 #define LBANN_DATA_READER_MESH_HPP
 
 #include "data_reader.hpp"
+#include "lbann/base.hpp"
+#include "lbann/lbann_stl.hpp"
 
 namespace lbann {
+class thread_pool;
 
 /**
  * Data reader for reading dumped mesh images.

include/lbann/data_readers/data_reader_mnist.hpp

@@ -30,6 +30,8 @@
 #define LBANN_DATA_READER_MNIST_HPP
 
 #include "data_reader_image.hpp"
+#include "lbann/base.hpp"
+#include "lbann/lbann_stl.hpp"
 
 namespace lbann {
 

include/lbann/data_readers/data_reader_nci.hpp

@@ -30,6 +30,7 @@
 #define LBANN_DATA_READER_NCI_HPP
 
 #include "data_reader_csv.hpp"
+#include "lbann/lbann_stl.hpp"
 
 #define NCI_HAS_HEADER
 

include/lbann/data_readers/data_reader_python.hpp

@@ -27,7 +27,16 @@
 #ifndef LBANN_DATA_READERS_PYTHON_HPP_INCLUDED
 #define LBANN_DATA_READERS_PYTHON_HPP_INCLUDED
 
+#include <El.hpp>
+
 #include "data_reader.hpp"
+#include "lbann/base.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/lbann_stl.hpp"
+
+namespace lbann {
+class thread_pool;
+}  // namespace lbann
 #ifdef LBANN_HAS_EMBEDDED_PYTHON
 #include "lbann/utils/python.hpp"
 

include/lbann/data_readers/data_reader_synthetic.hpp

@@ -30,6 +30,10 @@
 
 #include "data_reader.hpp"
 #include "lbann/utils/dim_helpers.hpp"
+#include "lbann/base.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/lbann_stl.hpp"
+#include "lbann/utils/exception.hpp"
 
 // Forward declaration
 class DataReaderSyntheticWhiteboxTester;

include/lbann/lbann_stl.hpp

@@ -0,0 +1,66 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014-2023, Lawrence Livermore National Security, LLC.
+// Produced at the Lawrence Livermore National Laboratory.
+// Written by the LBANN Research Team (B. Van Essen, et al.) listed in
+// the CONTRIBUTORS file. <lbann-dev@llnl.gov>
+//
+// LLNL-CODE-697807.
+// All rights reserved.
+//
+// This file is part of LBANN: Livermore Big Artificial Neural Network
+// Toolkit. For details, see http://software.llnl.gov/LBANN or
+// https://github.com/LLNL/LBANN.
+//
+// Licensed under the Apache License, Version 2.0 (the "Licensee"); you
+// may not use this file except in compliance with the License.  You may
+// obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the license.
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef LBANN_LBANN_STL_HPP_INCLUDED
+#define LBANN_LBANN_STL_HPP_INCLUDED
+
+#if defined(__cpp_modules) && __cpp_modules
+import std;
+#else
+// IWYU pragma: begin_exports
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdio>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <cctype>
+#include <deque>
+#include <fstream>
+#include <functional>
+#include <iosfwd>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <random>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+// IWYU pragma: end_exports
+#endif
+
+#endif // LBANN_LBANN_STL_HPP_INCLUDED

lbann_iwyu_mapping.imp

@@ -0,0 +1,40 @@
+[
+ { include: ["@<hydrogen/.*>", "private", "<El.hpp>", "public"] },
+ { include: ["@<h2/.*>", "private", "\"lbann/utils/h2_tmp.hpp\"", "public"] },
+
+ { include: ["@<cereal/types/.*>", "private", "\"lbann/utils/serialization/cereal_utils.hpp\"", "public"] },
+ { include: ["@<cereal/details/.*>", "private", "<cereal/cereal.hpp>", "public"] },
+
+ # Standard packages in the Conduit library, outside of the relay sublibrary should be in a standard header
+ { include: ["@<conduit/conduit_[^r].*>", "private", "<conduit/conduit.hpp>", "public"] },
+
+ { include: ["@<El/.*>", "private", "<El.hpp>", "public"] },
+
+ { include: ["@\"lbann/utils/serialization/.*\"", "public", "\"lbann/utils/serialize.hpp\"", "public"] },
+# { include: ["@<lbann/.*>", "private", "\"lbann/base.hpp\"", "public"] },
+ { include: ["\"lbann_config.hpp\"", "private", "\"lbann/base.hpp\"", "public"] },
+
+ # Grab all of the C++ implementation details and tie them into a common header
+ { include: ["@<__.*>", "private", "\"lbann/lbann_stl.hpp\"", "public"] },
+ { include: ["@<_.*>", "private", "\"lbann/lbann_stl.hpp\"", "public"] },
+
+ { symbol: ["hid_t", "private", "\"hdf5.h\"", "public"] },
+
+ # Handle obnoxious Python.h symbols
+ { symbol: ["PyObject_CallMethod", "private", "<Python.h>", "public"] },
+ { symbol: ["PyIter_Next", "private", "<Python.h>", "public"] },
+ { symbol: ["PyObject_CallObject", "private", "<Python.h>", "public"] },
+ { symbol: ["PyObject_GetIter", "private", "<Python.h>", "public"] },
+ { symbol: ["PyObject_Length", "private", "<Python.h>", "public"] },
+ { symbol: ["PyImport_ImportModule", "private", "<Python.h>", "public"] },
+ { symbol: ["PyList_Append", "private", "<Python.h>", "public"] },
+ { symbol: ["PyList_New", "private", "<Python.h>", "public"] },
+ { symbol: ["PyLong_AsLong", "private", "<Python.h>", "public"] },
+ { symbol: ["Py_BuildValue", "private", "<Python.h>", "public"] },
+ { symbol: ["PyObject_GetAttrString", "private", "<Python.h>", "public"] },
+ { symbol: ["PyObject_SetAttrString", "private", "<Python.h>", "public"] },
+ { symbol: ["Py_DECREF", "private", "<Python.h>", "public"] },
+ { symbol: ["PySys_GetObject", "private", "<Python.h>", "public"] },
+ { symbol: ["PyRun_SimpleString", "private", "<Python.h>", "public"] },
+
+]

src/data_readers/data_reader.cpp

@@ -26,23 +26,30 @@
 // lbann_data_reader .hpp .cpp - Input data base class for training, testing
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "lbann/comm_impl.hpp"
+#include <El.hpp>
+#include <cereal/cereal.hpp>
+#include <conduit/conduit.hpp>
+
 #include "lbann/data_readers/data_reader.hpp"
 #include "lbann/data_store/data_store_conduit.hpp"
 #include "lbann/execution_algorithms/sgd_execution_context.hpp"
-#include "lbann/io/persist.hpp"
 #include "lbann/io/persist_impl.hpp"
 #include "lbann/trainers/trainer.hpp"
-#include "lbann/utils/serialize.hpp"
 #include "lbann/utils/threads/thread_pool.hpp"
 #include "lbann/utils/timer.hpp"
-
-#include "conduit/conduit_node.hpp"
-
-#include <omp.h>
-#include <future>
+#include "lbann/base.hpp"
+#include "lbann/comm.hpp"
+#include "lbann/data_coordinator/data_coordinator.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/lbann_stl.hpp"
+#include "lbann/utils/argument_parser.hpp"
+#include "lbann/utils/enum_iterator.hpp"
+#include "lbann/utils/exception.hpp"
+#include "lbann/utils/options.hpp"
+#include "lbann/utils/random_number_generators.hpp"
 
 namespace lbann {
+class persist;
 
 #undef DEBUG
 //#define DEBUG

src/data_readers/data_reader_HDF5.cpp

@@ -24,13 +24,27 @@
 // permissions and limitations under the license.
 //
 /////////////////////////////////////////////////////////////////////////////////
-#include "conduit/conduit_relay_mpi.hpp"
+#include <El.hpp>
+#include <conduit/conduit.hpp>
+#include <conduit/conduit_relay_io.hpp>
+#include <conduit/conduit_relay_io_hdf5.hpp>
 
+#include "conduit/conduit_relay_mpi.hpp"
 #include "lbann/data_readers/data_reader_HDF5.hpp"
 #include "lbann/data_readers/data_reader_sample_list_impl.hpp"
 #include "lbann/data_readers/sample_list_impl.hpp"
 #include "lbann/data_readers/sample_list_open_files_impl.hpp"
 #include "lbann/utils/timer.hpp"
+#include "hdf5.h"
+#include "lbann/comm.hpp"
+#include "lbann/data_readers/data_reader_sample_list.hpp"
+#include "lbann/data_readers/sample_list_hdf5.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/data_store/data_store_conduit.hpp"
+#include "lbann/lbann_stl.hpp"
+#include "lbann/utils/argument_parser.hpp"
+#include "lbann/utils/exception.hpp"
+#include "lbann/utils/options.hpp"
 
 namespace lbann {
 namespace {

src/data_readers/data_reader_cifar10.cpp

@@ -27,7 +27,13 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "lbann/data_readers/data_reader_cifar10.hpp"
-#include "lbann/data_readers/sample_list_impl.hpp"
+
+#include <El.hpp>
+
+#include "lbann/data_readers/data_reader_image.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/transforms/transform_pipeline.hpp"
+#include "lbann/utils/exception.hpp"
 
 namespace lbann {
 

src/data_readers/data_reader_csv.cpp

@@ -26,12 +26,17 @@
 // lbann_data_reader_csv .hpp .cpp - generic_data_reader class for CSV files
 ////////////////////////////////////////////////////////////////////////////////
 
+#include <El.hpp>
+
 #include "lbann/comm_impl.hpp"
 #include "lbann/data_readers/data_reader_csv.hpp"
-#include "lbann/utils/options.hpp"
 #include "lbann/utils/threads/thread_pool.hpp"
-
-#include <unordered_set>
+#include "lbann/base.hpp"
+#include "lbann/comm.hpp"
+#include "lbann/data_readers/data_reader.hpp"
+#include "lbann/data_readers/utils/input_data_type.hpp"
+#include "lbann/lbann_stl.hpp"
+#include "lbann/utils/exception.hpp"
 
 namespace lbann {
 

src/data_readers/data_reader_image.cpp

@@ -26,17 +26,28 @@
 // data_reader_image .hpp .cpp - generic data reader class for image dataset
 ////////////////////////////////////////////////////////////////////////////////
 
+#include <cereal/archives/binary.hpp>
+#include <cereal/cereal.hpp>
+#include <conduit/conduit.hpp>
+
 #include "lbann/comm_impl.hpp"
 #include "lbann/data_readers/data_reader_image.hpp"
 #include "lbann/data_readers/sample_list_impl.hpp"
 #include "lbann/data_store/data_store_conduit.hpp"
 #include "lbann/utils/file_utils.hpp"
 #include "lbann/utils/lbann_library.hpp"
-#include "lbann/utils/threads/thread_utils.hpp"
 #include "lbann/utils/timer.hpp"
 #include "lbann/utils/vectorwrapbuf.hpp"
-
-#include <fstream>
+#include "lbann/base.hpp"
+#include "lbann/comm.hpp"
+#include "lbann/data_readers/data_reader.hpp"
+#include "lbann/data_readers/sample_list.hpp"
+#include "lbann/lbann_stl.hpp"
+#include "lbann/transforms/transform_pipeline.hpp"
+#include "lbann/utils/argument_parser.hpp"
+#include "lbann/utils/exception.hpp"
+#include "lbann/utils/options.hpp"
+#include "lbann/utils/threads/thread_pool.hpp"
 
 namespace lbann {