no more xtensor, futures carying dtype and rank

Author: fschlimb

.gitmodules

@@ -1,21 +1,6 @@
 [submodule "third_party/bitsery"]
 	path = third_party/bitsery
 	url = https://github.com/fraillt/bitsery
-[submodule "third_party/xtensor"]
-	path = third_party/xtensor
-	url = https://github.com/xtensor-stack/xtensor
-[submodule "third_party/xtensor-blas"]
-	path = third_party/xtensor-blas
-	url = https://github.com/xtensor-stack/xtensor-blas
-[submodule "third_party/xsimd"]
-	path = third_party/xsimd
-	url = https://github.com/xtensor-stack/xsimd
-[submodule "third_party/xtl"]
-	path = third_party/xtl
-	url = https://github.com/xtensor-stack/xtl
 [submodule "third_party/mlir-extensions"]
 	path = third_party/mlir-extensions
 	url = https://github.com/intel/mlir-extensions
-[submodule "third_party/dpcomp"]
-	path = third_party/dpcomp
-	url = https://github.com/IntelPython/dpcomp.git

CMakeLists.txt

@@ -87,14 +87,10 @@ set(MyCppSources ${MyCppSources} ${PROJECT_SOURCE_DIR}/src/jit/mlir.cpp ${P2C_HP
 
 pybind11_add_module(_ddptensor MODULE ${MyCppSources})
 
-target_compile_definitions(_ddptensor PRIVATE XTENSOR_USE_XSIMD=1 XTENSOR_USE_TBB=1 USE_MKL=1 DDPT_2TYPES=1)
+target_compile_definitions(_ddptensor PRIVATE USE_MKL=1 DDPT_2TYPES=1)
 target_compile_options(_ddptensor PRIVATE "-ftemplate-backtrace-limit=0")
 target_include_directories(_ddptensor PRIVATE
   ${PROJECT_SOURCE_DIR}/src/include
-  ${PROJECT_SOURCE_DIR}/third_party/xtl/include
-  ${PROJECT_SOURCE_DIR}/third_party/xsimd/include
-  ${PROJECT_SOURCE_DIR}/third_party/xtensor-blas/include
-  ${PROJECT_SOURCE_DIR}/third_party/xtensor/include
   ${PROJECT_SOURCE_DIR}/third_party/bitsery/include
   ${MPI_INCLUDE_PATH}
   $ENV{MKLROOT}/include

src/Creator.cpp

@@ -1,12 +1,13 @@
 #include "ddptensor/Creator.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
 #include "ddptensor/Deferred.hpp"
 #include "ddptensor/Factory.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 
 #include <imex/Dialect/PTensor/IR/PTensorOps.h>
 #include <mlir/IR/Builders.h>
 
+#if 0
 namespace x {
 
     template<typename T>
@@ -63,6 +64,7 @@ namespace x {
         }
     }; // class creatorx
 } // namespace x
+#endif // if 0
 
 struct DeferredFromShape : public Deferred
 {
@@ -72,14 +74,17 @@ struct DeferredFromShape : public Deferred
 
     DeferredFromShape() = default;
     DeferredFromShape(CreatorId op, const shape_type & shape, DTypeId dtype)
-        : _shape(shape), _dtype(dtype), _op(op)
+        : Deferred(dtype, shape.size()),
+          _shape(shape), _dtype(dtype), _op(op)
     {}
 
     void run()
     {
-        set_value(std::move(TypeDispatch<x::Creator>(_dtype, _op, _shape)));
+        // set_value(std::move(TypeDispatch<x::Creator>(_dtype, _op, _shape)));
     }
 
+    // FIXME mlir
+
     FactoryId factory() const
     {
         return F_FROMSHAPE;
@@ -107,15 +112,18 @@ struct DeferredFull : public Deferred
 
     DeferredFull() = default;
     DeferredFull(const shape_type & shape, PyScalar val, DTypeId dtype)
-        : _shape(shape), _val(val), _dtype(dtype)
+        : Deferred(dtype, shape.size()),
+          _shape(shape), _val(val), _dtype(dtype)
     {}
 
     void run()
     {
-        auto op = FULL;
-        set_value(std::move(TypeDispatch<x::Creator>(_dtype, op, _shape, _val)));
+        // auto op = FULL;
+        // set_value(std::move(TypeDispatch<x::Creator>(_dtype, op, _shape, _val)));
     }
 
+    // FIXME mlir
+
     FactoryId factory() const
     {
         return F_FULL;
@@ -139,11 +147,11 @@ ddptensor * Creator::full(const shape_type & shape, const py::object & val, DTyp
 struct DeferredArange : public Deferred
 {
     uint64_t _start, _end, _step;
-    DTypeId _dtype;
 
     DeferredArange() = default;
     DeferredArange(uint64_t start, uint64_t end, uint64_t step, DTypeId dtype)
-        : _start(start), _end(end), _step(step), _dtype(dtype)
+        : Deferred(dtype, 1),
+          _start(start), _end(end), _step(step)
     {}
 
     void run() override
@@ -153,21 +161,20 @@ struct DeferredArange : public Deferred
 
     ::mlir::Value generate_mlir(::mlir::OpBuilder & builder, ::mlir::Location loc, jit::IdValueMap & ivm) override
     {
-        // FIXME the type of the result is hard-coded to uint64_t
         // create start, stop and step
         auto start = jit::createI64(loc, builder, _start);
         auto end = jit::createI64(loc, builder, _end);
         auto step = jit::createI64(loc, builder, _step);
         // create arange
         auto dtype = builder.getI64Type();
+        assert(_dtype == INT64 || _dtype == UINT64); // FIXME
         llvm::SmallVector<int64_t> shape(1, -1); //::mlir::ShapedType::kDynamicSize);
         auto artype = ::imex::ptensor::PTensorType::get(builder.getContext(), ::mlir::RankedTensorType::get(shape, dtype), true);
         auto ar = builder.create<::imex::ptensor::ARangeOp>(loc, artype, start, end, step, true);
         auto setter = [this](uint64_t rank, void *allocated, void *aligned, intptr_t offset, const intptr_t * sizes, const intptr_t * strides) {
-            // FIXME GC assert(allocated == aligned);
             assert(rank == 1);
             assert(strides[0] == 1);
-            this->set_value(std::move(x::operatorx<uint64_t>::mk_tx(rank, allocated, aligned, offset, sizes, strides)));
+            this->set_value(std::move(mk_tnsr(_dtype, rank, allocated, aligned, offset, sizes, strides)));
         };
         ivm[_guid] = {ar, setter};
         return ar;

src/Deferred.cpp

@@ -11,6 +11,7 @@
 #include <mlir/Dialect/LLVMIR/LLVMDialect.h>
 
 #include <iostream>
+#include <unordered_set>
 
 static tbb::concurrent_bounded_queue<Runable::ptr_type> _deferred;
 
@@ -27,7 +28,7 @@ void _dist(const Runable * p)
 
 Deferred::future_type Deferred::get_future()
 {
-    return {std::move(tensor_i::promise_type::get_future().share()), _guid};
+    return {std::move(promise_type::get_future().share()), _guid, _dtype, _rank};
 }
 
 Deferred::future_type defer_tensor(Runable::ptr_type && _d, bool is_global)
@@ -36,7 +37,7 @@ Deferred::future_type defer_tensor(Runable::ptr_type && _d, bool is_global)
     if(!d) throw std::runtime_error("Expected Deferred Tensor promise");
     if(is_global) {
         _dist(d);
-        d->_guid = Registry::get_guid();
+        d->set_guid(Registry::get_guid());
     }
     auto f = d->get_future();
     Registry::put(f);
@@ -59,6 +60,25 @@ void Runable::fini()
     _deferred.clear();
 }
 
+#if 0
+class DepManager
+{
+private:
+    IdValueMap _ivm;
+    std::unordered_set<id_type> _args;
+public:
+    ::mlir::Value getDependent(i::mlir::OpBuilder & builder, d_type guid)
+    {
+        if(auto d = _ivm.find(guid); d == _ivm.end()) {
+            _func.insertArg
+            _ivm[guid] = {val, {}}
+        } else {
+            return d->second.first;
+        }
+    }
+};
+#endif
+
 void process_promises()
 {
     jit::JIT jit;

src/EWBinOp.cpp

@@ -1,17 +1,15 @@
 // SPDX-License-Identifier: BSD-3-Clause
 
-#include <xtensor/xview.hpp>
-using namespace xt::placeholders;
 #include "ddptensor/EWBinOp.hpp"
 #include "ddptensor/LinAlgOp.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
 #include "ddptensor/Factory.hpp"
 #include "ddptensor/Registry.hpp"
 #include "ddptensor/Creator.hpp"
 #include "ddptensor/TypePromotion.hpp"
 #include "ddptensor/CollComm.hpp"
 #include "ddptensor/Chunker.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 
 #include <imex/Dialect/PTensor/IR/PTensorOps.h>
 #include <mlir/IR/Builders.h>
@@ -45,6 +43,7 @@ using namespace xt::placeholders;
 // above regions and apply the op. All buffers/views get linearized/flattened.
 // #######################################################################################
 
+#if 0
 namespace x {
 
     // @return true if operation returs bool, false otherwise
@@ -371,6 +370,7 @@ namespace x {
         }
     };
 } // namespace x
+#endif // if 0
 
 // convert id of our binop to id of imex::ptensor binop
 static ::imex::ptensor::EWBinOpId ddpt2mlir(const EWBinOpId bop)
@@ -443,7 +443,8 @@ struct DeferredEWBinOp : public Deferred
 
     DeferredEWBinOp() = default;
     DeferredEWBinOp(EWBinOpId op, const tensor_i::future_type & a, const tensor_i::future_type & b)
-        : _a(a.id()), _b(b.id()), _op(op)
+        : Deferred(a.dtype(), a.rank()),
+          _a(a.id()), _b(b.id()), _op(op)
     {}
 
     void run() override
@@ -457,12 +458,11 @@ struct DeferredEWBinOp : public Deferred
 
     ::mlir::Value generate_mlir(::mlir::OpBuilder & builder, ::mlir::Location loc, jit::IdValueMap & ivm) override
     {
-        // FIXME the type of the result is hard-coded to uint64_t
+        // FIXME the type of the result is based on a only
         auto rtyp = ivm[_a].first.getType();
         auto ewbo = builder.create<::imex::ptensor::EWBinOp>(loc, rtyp, builder.getI32IntegerAttr(ddpt2mlir(_op)), ivm[_a].first, ivm[_b].first);
         auto setter = [this](uint64_t rank, void *allocated, void *aligned, intptr_t offset, const intptr_t * sizes, const intptr_t * strides) {
-            // FIXME GC assert(allocated == aligned);
-            this->set_value(std::move(x::operatorx<uint64_t>::mk_tx(rank, allocated, aligned, offset, sizes, strides)));
+            this->set_value(std::move(mk_tnsr(_dtype, rank, allocated, aligned, offset, sizes, strides)));
         };
         ivm[_guid] = {ewbo, setter};
         return ewbo;

src/EWUnyOp.cpp

@@ -1,8 +1,9 @@
 #include "ddptensor/EWUnyOp.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 #include "ddptensor/Factory.hpp"
 
+#if 0
 namespace x {
 
     class EWUnyOp
@@ -108,6 +109,7 @@ namespace x {
 
     };
 } //namespace x
+#endif // if 0
 
 struct DeferredEWUnyOp : public Deferred
 {
@@ -121,8 +123,8 @@ struct DeferredEWUnyOp : public Deferred
 
     void run()
     {
-        const auto a = std::move(Registry::get(_a).get());
-        set_value(std::move(TypeDispatch<x::EWUnyOp>(a, _op)));
+        // const auto a = std::move(Registry::get(_a).get());
+        // set_value(std::move(TypeDispatch<x::EWUnyOp>(a, _op)));
     }
 
     FactoryId factory() const

src/IEWBinOp.cpp

@@ -1,9 +1,10 @@
 #include "ddptensor/IEWBinOp.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 #include "ddptensor/Factory.hpp"
 #include "ddptensor/Creator.hpp"
 
+#if 0
 namespace x {
 
     class IEWBinOp
@@ -74,6 +75,7 @@ namespace x {
 
     };
 } // namespace x
+#endif // if 0
 
 struct DeferredIEWBinOp : public Deferred
 {
@@ -88,9 +90,9 @@ struct DeferredIEWBinOp : public Deferred
 
     void run()
     {
-        const auto a = std::move(Registry::get(_a).get());
-        const auto b = std::move(Registry::get(_b).get());
-        set_value(std::move(TypeDispatch<x::IEWBinOp>(a, b, _op)));
+        // const auto a = std::move(Registry::get(_a).get());
+        // const auto b = std::move(Registry::get(_b).get());
+        // set_value(std::move(TypeDispatch<x::IEWBinOp>(a, b, _op)));
     }
 
     FactoryId factory() const

src/IO.cpp

@@ -2,6 +2,7 @@
 #include "ddptensor/SetGetItem.hpp"
 #include "ddptensor/TypeDispatch.hpp"
 #include "ddptensor/Factory.hpp"
+#include "ddptensor/Transceiver.hpp"
 
 using promise_type = std::promise<py::object>;
 using future_type = std::shared_future<py::object>;

src/LinAlgOp.cpp

@@ -2,10 +2,10 @@
 //#include <mkl.h>
 #include "ddptensor/LinAlgOp.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 #include "ddptensor/Factory.hpp"
-#include "xtensor-blas/xlinalg.hpp"
 
+#if 0
 namespace x {
 
     template<typename T> struct TGEMM;
@@ -109,6 +109,7 @@ namespace x {
         }
     };
 }
+#endif // if 0
 
 struct DeferredLinAlgOp : public Deferred
  {
@@ -123,9 +124,9 @@ struct DeferredLinAlgOp : public Deferred
 
     void run()
     {
-        const auto a = std::move(Registry::get(_a).get());
-        const auto b = std::move(Registry::get(_b).get());
-        set_value(std::move(TypeDispatch<x::LinAlgOp>(a, b, _axis)));
+        //const auto a = std::move(Registry::get(_a).get());
+        //const auto b = std::move(Registry::get(_b).get());
+        //set_value(std::move(TypeDispatch<x::LinAlgOp>(a, b, _axis)));
     }
 
     FactoryId factory() const

src/ManipOp.cpp

@@ -1,10 +1,11 @@
 #include <mpi.h>
 #include "ddptensor/ManipOp.hpp"
 #include "ddptensor/TypeDispatch.hpp"
-#include "ddptensor/x.hpp"
+#include "ddptensor/DDPTensorImpl.hpp"
 #include "ddptensor/CollComm.hpp"
 #include "ddptensor/Factory.hpp"
 
+#if 0
 namespace x {
 
     class ManipOp
@@ -23,6 +24,7 @@ namespace x {
         }
     };
 }
+#endif // if 0
 
 struct DeferredManipOp : public Deferred
 {
@@ -36,8 +38,8 @@ struct DeferredManipOp : public Deferred
 
     void run()
     {
-        const auto a = std::move(Registry::get(_a).get());
-        set_value(std::move(TypeDispatch<x::ManipOp>(a, _shape)));
+        //const auto a = std::move(Registry::get(_a).get());
+        //set_value(std::move(TypeDispatch<x::ManipOp>(a, _shape)));
     }
 
     FactoryId factory() const