adding random.uniform and seed, better use of cmake (MKL) and more types and modern dispatch

fschlimb · fschlimb · commit aadf34e9884d · 2022-03-11T05:04:48.000-06:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -16,12 +16,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Find Python3 and NumPy
 find_package(Python3 COMPONENTS Interpreter Development.Module NumPy REQUIRED)
-
-find_package(Python COMPONENTS Interpreter Development)
 find_package(pybind11 CONFIG)
 find_package(MPI REQUIRED)
-include_directories(SYSTEM ${MPI_INCLUDE_PATH} ${pybind11_INCLUDE_DIRS})
 
+set(MKL_LIBRARIES -L$ENV{MKLROOT}/lib -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -lpthread -lrt -ldl -lm)
 # Use -fPIC even if statically compiled
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
@@ -34,6 +32,6 @@ FILE(GLOB MyCppSources ${PROJECT_SOURCE_DIR}/src/*.cpp ${PROJECT_SOURCE_DIR}/src
 #add_library(_ddptensor MODULE ${MyCppSources})
 pybind11_add_module(_ddptensor THIN_LTO ${MyCppSources})
 
-target_include_directories(_ddptensor PRIVATE ${PROJECT_SOURCE_DIR}/src/include ${PROJECT_SOURCE_DIR}/third_party/xtl/include ${PROJECT_SOURCE_DIR}/third_party/xsimd/include ${PROJECT_SOURCE_DIR}/third_party/xtensor-blas/include ${PROJECT_SOURCE_DIR}/third_party/xtensor/include ${PROJECT_SOURCE_DIR}/third_party/bitsery/include)
-
-target_link_libraries(_ddptensor PRIVATE ${MPI_C_LIBRARIES})
+target_compile_definitions(_ddptensor PRIVATE USE_MKL=1 XTENSOR_USE_XSIMD=1 XTENSOR_USE_OPENMP=1) # DDPT_2TYPES=1)
+target_include_directories(_ddptensor PRIVATE ${PROJECT_SOURCE_DIR}/src/include ${PROJECT_SOURCE_DIR}/third_party/xtl/include ${PROJECT_SOURCE_DIR}/third_party/xsimd/include ${PROJECT_SOURCE_DIR}/third_party/xtensor-blas/include ${PROJECT_SOURCE_DIR}/third_party/xtensor/include ${PROJECT_SOURCE_DIR}/third_party/bitsery/include ${MPI_INCLUDE_PATH} $ENV{MKLROOT}/include ${pybind11_INCLUDE_DIRS})
+target_link_libraries(_ddptensor PRIVATE ${MPI_C_LIBRARIES} ${MKL_LIBRARIES})
diff --git a/ddptensor/__init__.py b/ddptensor/__init__.py
@@ -23,7 +23,7 @@
 for op in api.ew_binary_ops:
     OP = op.upper()
     exec(
-        f"{op} = lambda this, other: dtensor(_cdt.EWBinOp.op(_cdt.{OP}, this._t, other._t))"  # if isinstance(other, ddptensor) else other, False))"
+        f"{op} = lambda this, other: dtensor(_cdt.EWBinOp.op(_cdt.{OP}, this._t, other._t if isinstance(other, ddptensor) else other))"
     )
 
 for op in api.ew_unary_ops:
diff --git a/ddptensor/array_api.py b/ddptensor/array_api.py
@@ -4,6 +4,20 @@
 https://data-apis.org/array-api/latest
 """
 
+dtypes = [
+    "float32",
+    "float64",
+    "int8",
+    "int16",
+    "int32",
+    "int64",
+    "uint8",
+    "uint16",
+    "uint32",
+    "uint64",
+    "bool",
+]
+
 creators = [
     "arange",  # (start, /, stop=None, step=1, *, dtype=None, device=None)
     "asarray",  # (obj, /, *, dtype=None, device=None, copy=None)
@@ -62,6 +76,8 @@
     "tan",  # (x, /)
     "tanh",  # (x, /)
     "trunc",  # (x, /)
+    # non standard from here
+    "erf",  # (x, /)
 ]
 
 ew_binary_methods_inplace = [
diff --git a/ddptensor/ddptensor.py b/ddptensor/ddptensor.py
@@ -20,7 +20,7 @@ def __repr__(self):
     for method in api.ew_binary_methods:
         METHOD = method.upper()
         exec(
-            f"{method} = lambda self, other: dtensor(_cdt.EWBinOp.op(_cdt.{METHOD}, self._t, other._t))" # if isinstance(other, dtensor) else other, True))"
+            f"{method} = lambda self, other: dtensor(_cdt.EWBinOp.op(_cdt.{METHOD}, self._t, other._t if isinstance(other, dtensor) else other))"
         )
 
     for method in api.ew_binary_methods_inplace:
diff --git a/ddptensor/random.py b/ddptensor/random.py
@@ -0,0 +1,8 @@
+from . import _ddptensor as _cdt
+from . ddptensor import dtensor
+
+def uniform(low, high, size, dtype=_cdt.float64):
+    return dtensor(_cdt.Random.uniform(dtype, size, low, high))
+
+def seed(s):
+    _cdt.Random.seed(s)
diff --git a/scripts/code_gen.py b/scripts/code_gen.py
@@ -13,8 +13,16 @@
 namespace py = pybind11;
 """)
 
+# dtypes must go first
+print("enum DType {")
+for x in api.dtypes:
+    print(f"    DT_{x.upper()},")
+print("    DTYPE_LAST")
+print("};\n")
+
 print("enum CreatorId : int {")
 for x in api.creators:
+    x = x + " = DTYPE_LAST" if x == api.creators[0] else x
     print(f"    {x.upper()},")
 print("    CREATOR_LAST")
 print("};\n")
@@ -51,6 +59,11 @@
 
 print("static void def_enums(py::module_ & m)\n{")
 
+print('    py::enum_<DType>(m, "dtype")')
+for x in api.dtypes:
+    print(f'        .value("{x}", DT_{x.upper()})')
+print("        .export_values();\n")
+
 print('    py::enum_<CreatorId>(m, "CreatorId")')
 for x in api.creators:
     print(f'        .value("{x.upper()}", {x.upper()})')
diff --git a/setup.py b/setup.py
@@ -53,7 +53,7 @@ def build_cmake(self, ext):
 setup(name="ddptensor",
       version="0.1",
       description="Distributed Tensor and more",
-      packages=["ddptensor", "ddptensor.numpy", "ddptensor.torch"],
+      packages=["ddptensor"],  #, "ddptensor.numpy", "ddptensor.torch"],
       ext_modules=[CMakeExtension('ddptensor/_ddptensor')],
       cmdclass=dict(
           # Enable the CMakeExtension entries defined above
diff --git a/src/EWBinOp.cpp b/src/EWBinOp.cpp
@@ -115,5 +115,5 @@ namespace x {
     
 tensor_i::ptr_type EWBinOp::op(EWBinOpId op, x::DPTensorBaseX::ptr_type a, x::DPTensorBaseX::ptr_type b)
 {
-    return TypeDispatch2<x::EWBinOp>(a, b, op);
+    return TypeDispatch<x::EWBinOp>(a, b, op);
 }
diff --git a/src/EWUnyOp.cpp b/src/EWUnyOp.cpp
@@ -74,6 +74,8 @@ namespace x {
                 return operatorx<T>::mk_tx_(a_ptr, xt::tanh(a));
             case TRUNC:
                 return operatorx<T>::mk_tx_(a_ptr, xt::trunc(a));
+            case ERF:
+                return operatorx<T>::mk_tx_(a_ptr, xt::erf(a));
             case __NEG__:
             case NEGATIVE:
             case __POS__:
@@ -98,5 +100,5 @@ namespace x {
 
 tensor_i::ptr_type EWUnyOp::op(EWUnyOpId op, x::DPTensorBaseX::ptr_type a)
 {
-    return TypeDispatch2<x::EWUnyOp>(a, op);
+    return TypeDispatch<x::EWUnyOp>(a, op);
 }
diff --git a/src/IEWBinOp.cpp b/src/IEWBinOp.cpp
@@ -64,5 +64,5 @@ namespace x {
 
 void IEWBinOp::op(IEWBinOpId op, x::DPTensorBaseX::ptr_type a, x::DPTensorBaseX::ptr_type b)
 {
-    TypeDispatch2<x::IEWBinOp>(a, b, op);
+    TypeDispatch<x::IEWBinOp>(a, b, op);
 }
diff --git a/src/Random.cpp b/src/Random.cpp
@@ -0,0 +1,38 @@
+#include <xtensor/xrandom.hpp>
+#include "ddptensor/Random.hpp"
+#include "ddptensor/x.hpp"
+#include "ddptensor/Operations.hpp"
+
+using ptr_type = tensor_i::ptr_type;
+
+namespace x {
+
+    template<typename T>
+    struct Rand
+    {
+        template<typename L, typename U>
+        static ptr_type op(const shape_type & shape, const L & lower, const U & upper)
+        {
+            if constexpr (std::is_floating_point<T>::value) {
+                PVSlice pvslice(shape);
+                return operatorx<T>::mk_tx(std::move(pvslice), std::move(xt::random::rand(shape, to_native<T>(lower), to_native<T>(upper))));
+            }
+        }
+    };
+}
+
+ptr_type Random::rand(DType dtype, const shape_type & shape, const py::object & lower, const py::object & upper)
+{
+    switch(dtype) {
+    case DT_FLOAT64:
+        return x::Rand<double>::op(shape, lower, upper);
+    case DT_FLOAT32:
+        return x::Rand<double>::op(shape, lower, upper);
+    }
+    throw std::runtime_error("rand: dtype must be a floating point type");
+}
+
+void Random::seed(uint64_t s)
+{
+    xt::random::seed(s);
+}
diff --git a/src/ReduceOp.cpp b/src/ReduceOp.cpp
@@ -56,5 +56,5 @@ namespace x {
 
 tensor_i::ptr_type ReduceOp::op(ReduceOpId op, x::DPTensorBaseX::ptr_type a, const dim_vec_type & dim)
 {
-    return TypeDispatch2<x::ReduceOp>(a, op, dim);
+    return TypeDispatch<x::ReduceOp>(a, op, dim);
 }
diff --git a/src/SetGetItem.cpp b/src/SetGetItem.cpp
@@ -3,39 +3,35 @@
 
 namespace x {
 
-    template<typename T>
     class GetItem
     {
     public:
         using ptr_type = DPTensorBaseX::ptr_type;
 
-        static ptr_type op(const ptr_type & a_ptr, const NDSlice & slice)
+        template<typename T>
+        static ptr_type op(const NDSlice & slice, const std::shared_ptr<DPTensorX<T>> & a_ptr)
         {
-            const auto _a = dynamic_cast<DPTensorX<T>*>(a_ptr.get());
-            if(!_a )
-                throw std::runtime_error("Invalid array object: could not dynamically cast");
-            auto nd = _a->shape().size();
+            auto nd = a_ptr->shape().size();
             if(nd != slice.ndims())
                 throw std::runtime_error("Index dimensionality must match array dimensionality");
 
-            return operatorx<T>::mk_tx(*_a, slice);
+            return operatorx<T>::mk_tx(*a_ptr.get(), slice);
         }
     };
 
-    template<typename T>
     class SetItem
     {
     public:
         using ptr_type = DPTensorBaseX::ptr_type;
 
         // copy data from val into (*dest)[slice]
         // this is a non-collective call.
-        template<typename X, typename U>
-        static void _set_slice(X && dest, const PVSlice & org_slice, const NDSlice & dest_slice, const DPTensorX<U> & val, const NDSlice & val_slice)
+        template<typename T, typename X, typename U>
+        static void _set_slice(X && dest, const PVSlice & org_slice, const NDSlice & dest_slice, const std::shared_ptr<DPTensorX<U>> & val, const NDSlice & val_slice)
         // (DPTensorX<T> & dest, const NDSlice & dest_slice, const DPTensorX<U> & val, const NDSlice & val_slice)
         {
             // const PVSlice & org_slice = dest.slice();
-            std::cerr << "_set_slice " << org_slice << " " << dest_slice << " " << val.slice() << " " << val_slice << std::endl;
+            std::cerr << "_set_slice " << org_slice << " " << dest_slice << " " << val->slice() << " " << val_slice << std::endl;
             auto nd = org_slice.ndims();
             // if(dest.owner() == REPLICATED && nd > 0)
             //     std::cerr << "Warning: __setitem__ on replicated data updates local tile only" << std::endl;
@@ -48,13 +44,13 @@ namespace x {
             PVSlice g_slc_view(org_slice, dest_slice);
             std::cerr << "g_slice: " << g_slc_view.slice() << std::endl;
             // Create a view into val
-            PVSlice needed_val_view(val.slice(), val_slice);
-            std::cerr << "needed_val_view: " << needed_val_view.slice() << " (was " << val.slice().slice() << ")" << std::endl;
+            PVSlice needed_val_view(val->slice(), val_slice);
+            std::cerr << "needed_val_view: " << needed_val_view.slice() << " (was " << val->slice().slice() << ")" << std::endl;
 
             // we can now compute which ranks actually hold which piece of the data from val that we need locally
             for(rank_type i=0; i<theTransceiver->nranks(); ++i ) {
                 // get local view into val
-                PVSlice val_local_view(val.slice(), i);
+                PVSlice val_local_view(val->slice(), i);
                 std::cerr << i << " val_local_view: " << val_local_view.slice() << std::endl;
                 NDSlice curr_needed_val_slice = needed_val_view.slice_of_rank(i);
                 std::cerr << i << " curr_needed_val_slice: " << curr_needed_val_slice << std::endl;
@@ -72,14 +68,14 @@ namespace x {
                         // copy locally
                         std::cerr << "local copy\n";
                         auto to_v   = xt::strided_view(dest/*.xarray()*/, to_xt(my_curr_local_slice));
-                        auto from_v = xt::strided_view(val.xarray(), to_xt(curr_local_val_slice));
+                        auto from_v = xt::strided_view(val->xarray(), to_xt(curr_local_val_slice));
                         std::cerr << "to: " << to_v << std::endl << "from: " << from_v << std::endl;
                         to_v = from_v;
                     } else {
                         // pull slice directly into new array
                         xt::xarray<U> from_a = xt::empty<U>(curr_local_val_slice.shape());
                         from_a.fill(static_cast<U>(4711));
-                        theMediator->pull(i, val, curr_local_val_slice, from_a.data());
+                        theMediator->pull(i, *val.get(), curr_local_val_slice, from_a.data());
                         auto to_v = xt::strided_view(dest/*.xarray()*/, to_xt(my_curr_local_slice));
                         to_v = from_a;
                     }
@@ -90,40 +86,33 @@ namespace x {
         // FIXME We use a generic SPMD/PGAS mechanism to pull elements from remote
         // on all procs simultaneously.  Since __setitem__ is collective we could
         // implement a probaly more efficient mechanism which pushes data and/or using RMA.
-        static void op(ptr_type & a_ptr, const NDSlice & slice, const ptr_type & b_ptr)
+        template<typename A, typename B>
+        static void op(const NDSlice & slice, std::shared_ptr<DPTensorX<A>> a_ptr, const std::shared_ptr<DPTensorX<B>> & b_ptr)
         {
-            const auto _a = dynamic_cast<DPTensorX<T>*>(a_ptr.get());
-            const auto _b = dynamic_cast<DPTensorX<T>*>(b_ptr.get());
-            if(!_a || !_b)
-                throw std::runtime_error("Invalid array object: could not dynamically cast");
-
             // Use given slice to create a global view into orig array
-            PVSlice g_slc_view(_a->slice(), slice);
+            PVSlice g_slc_view(a_ptr->slice(), slice);
             std::cerr << "g_slice: " << g_slc_view.slice() << std::endl;
             NDSlice my_slice = g_slc_view.slice_of_rank();
             std::cerr << "my_slice: " << my_slice << std::endl;
             NDSlice my_norm_slice = g_slc_view.map_slice(my_slice);
             std::cerr << "my_norm_slice: " << my_norm_slice << std::endl;
-            NDSlice my_rel_slice = _a->slice().map_slice(my_slice);
+            NDSlice my_rel_slice = a_ptr->slice().map_slice(my_slice);
             std::cerr << "my_rel_slice: " << my_rel_slice << std::endl;
             
             theTransceiver->barrier();
-            _set_slice(_a->xarray(), _a->slice(),
-                       my_rel_slice, *_b, my_norm_slice);
+            _set_slice<A>(a_ptr->xarray(), a_ptr->slice(),
+                          my_rel_slice, b_ptr, my_norm_slice);
         }
     };
 
-    template<typename T>
     class SPMD
     {
     public:
         using ptr_type = DPTensorBaseX::ptr_type;
 
-        static py::object op(const ptr_type & a_ptr, const NDSlice & slice)
+        template<typename T>
+        static py::object op(const NDSlice & slice, const std::shared_ptr<DPTensorX<T>> & a_ptr)
         {
-            const auto _a = dynamic_cast<DPTensorX<T>*>(a_ptr.get());
-            if(!_a)
-                throw std::runtime_error("Invalid array object: could not dynamically cast");
             auto shp = slice.shape();
             auto sz = VPROD(shp);
             auto res = py::array_t<T>(sz);
@@ -132,7 +121,7 @@ namespace x {
             // Create dtensor without creating id: do not use create_dtensor
             // auto out = DPTensorX<T>(ax, PVSlice(shp, NOSPLIT));
             PVSlice slc{shp, NOSPLIT};
-            SetItem<T>::_set_slice(ax, slc, slc.slice(), *_a, slice);
+            SetItem::_set_slice<T>(ax, slc, slc.slice(), a_ptr, slice);
             std::cerr << ax << std::endl << py::str(res).cast<std::string>() << std::endl;
             // res.reshape(shp);
             return res;
@@ -143,15 +132,15 @@ namespace x {
 
 void SetItem::__setitem__(x::DPTensorBaseX::ptr_type a, const std::vector<py::slice> & v, x::DPTensorBaseX::ptr_type b)
 {
-    return TypeDispatch<x::SetItem>(a->dtype(), a, NDSlice(v), b);
+    return TypeDispatch<x::SetItem>(a, b, NDSlice(v));
 }
 
 tensor_i::ptr_type GetItem::__getitem__(x::DPTensorBaseX::ptr_type a, const std::vector<py::slice> & v)
 {
-    return TypeDispatch<x::GetItem>(a->dtype(), a, NDSlice(v));
+    return TypeDispatch<x::GetItem>(a, NDSlice(v));
 }
 
 py::object GetItem::get_slice(x::DPTensorBaseX::ptr_type a, const std::vector<py::slice> & v)
 {
-    return TypeDispatch<x::SPMD>(a->dtype(), a, NDSlice(v));
+    return TypeDispatch<x::SPMD>(a, NDSlice(v));
 }
diff --git a/src/ddptensor.cpp b/src/ddptensor.cpp
@@ -22,6 +22,7 @@ using namespace pybind11::literals; // to bring _a
 #include "ddptensor/MPITransceiver.hpp"
 #include "ddptensor/MPIMediator.hpp"
 #include "ddptensor/Operations.hpp"
+#include "ddptensor/Random.hpp"
 
 // #########################################################################
 // The following classes are wrappers bridging pybind11 defs to TypeDispatch
@@ -42,7 +43,7 @@ void fini()
     delete theTransceiver;
     theTransceiver = nullptr;
 }
-    
+
 // #########################################################################
 // Finally our Python module
 PYBIND11_MODULE(_ddptensor, m) {
@@ -53,18 +54,6 @@ PYBIND11_MODULE(_ddptensor, m) {
 
     def_enums(m);
 
-    py::enum_<DType>(m, "dtype")
-        .value("float64", DT_FLOAT64)
-        .value("float32", DT_FLOAT32)
-        .value("int64", DT_INT64)
-        .value("int32", DT_INT32)
-        .value("int16", DT_INT16)
-        .value("uint64", DT_UINT64)
-        .value("uint32", DT_UINT32)
-        .value("uint16", DT_UINT16)
-        .value("bool", DT_BOOL)
-        .export_values();
-
     m.def("fini", &fini)
         .def("myrank", &myrank)
         .def("_get_slice", &GetItem::get_slice);
@@ -99,6 +88,10 @@ PYBIND11_MODULE(_ddptensor, m) {
         .def("__getitem__", &GetItem::__getitem__)
         .def("__setitem__", &SetItem::__setitem__);
 
+    py::class_<Random>(m, "Random")
+        .def("seed", &Random::seed)
+        .def("uniform", &Random::rand);
+
 #if 0
     py::class_<dtensor>(m, "dtensor")
         .def(py::init<const shape_type &, DType>())
diff --git a/src/include/ddptensor/Operations.hpp b/src/include/ddptensor/Operations.hpp
diff --git a/src/include/ddptensor/Random.hpp b/src/include/ddptensor/Random.hpp
diff --git a/src/include/ddptensor/UtilsAndTypes.hpp b/src/include/ddptensor/UtilsAndTypes.hpp
diff --git a/src/include/ddptensor/p2c_ids.hpp b/src/include/ddptensor/p2c_ids.hpp
diff --git a/src/include/ddptensor/x.hpp b/src/include/ddptensor/x.hpp
diff --git a/test/test_x.py b/test/test_x.py

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@`
`23`	`23`	`for op in api.ew_binary_ops:`
`24`	`24`	`OP = op.upper()`
`25`	`25`	`exec(`
`26`		`- f"{op} = lambda this, other: dtensor(_cdt.EWBinOp.op(_cdt.{OP}, this._t, other._t))" # if isinstance(other, ddptensor) else other, False))"`
	`26`	`+ f"{op} = lambda this, other: dtensor(_cdt.EWBinOp.op(_cdt.{OP}, this._t, other._t if isinstance(other, ddptensor) else other))"`
`27`	`27`	`)`
`28`	`28`
`29`	`29`	`for op in api.ew_unary_ops:`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ def __repr__(self):`
`20`	`20`	`for method in api.ew_binary_methods:`
`21`	`21`	`METHOD = method.upper()`
`22`	`22`	`exec(`
`23`		`- f"{method} = lambda self, other: dtensor(_cdt.EWBinOp.op(_cdt.{METHOD}, self._t, other._t))" # if isinstance(other, dtensor) else other, True))"`
	`23`	`+ f"{method} = lambda self, other: dtensor(_cdt.EWBinOp.op(_cdt.{METHOD}, self._t, other._t if isinstance(other, dtensor) else other))"`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`for method in api.ew_binary_methods_inplace:`
Original file line number	Diff line number	Diff line change
`@@ -115,5 +115,5 @@ namespace x {`
`115`	`115`
`116`	`116`	`tensor_i::ptr_type EWBinOp::op(EWBinOpId op, x::DPTensorBaseX::ptr_type a, x::DPTensorBaseX::ptr_type b)`
`117`	`117`	`{`
`118`		`- return TypeDispatch2<x::EWBinOp>(a, b, op);`
	`118`	`+ return TypeDispatch<x::EWBinOp>(a, b, op);`
`119`	`119`	`}`
Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,8 @@ namespace x {`
`74`	`74`	`return operatorx<T>::mk_tx_(a_ptr, xt::tanh(a));`
`75`	`75`	`case TRUNC:`
`76`	`76`	`return operatorx<T>::mk_tx_(a_ptr, xt::trunc(a));`
	`77`	`+ case ERF:`
	`78`	`+ return operatorx<T>::mk_tx_(a_ptr, xt::erf(a));`
`77`	`79`	`case __NEG__:`
`78`	`80`	`case NEGATIVE:`
`79`	`81`	`case __POS__:`
`@@ -98,5 +100,5 @@ namespace x {`
`98`	`100`
`99`	`101`	`tensor_i::ptr_type EWUnyOp::op(EWUnyOpId op, x::DPTensorBaseX::ptr_type a)`
`100`	`102`	`{`
`101`		`- return TypeDispatch2<x::EWUnyOp>(a, op);`
	`103`	`+ return TypeDispatch<x::EWUnyOp>(a, op);`
`102`	`104`	`}`
Original file line number	Diff line number	Diff line change
`@@ -64,5 +64,5 @@ namespace x {`
`64`	`64`
`65`	`65`	`void IEWBinOp::op(IEWBinOpId op, x::DPTensorBaseX::ptr_type a, x::DPTensorBaseX::ptr_type b)`
`66`	`66`	`{`
`67`		`- TypeDispatch2<x::IEWBinOp>(a, b, op);`
	`67`	`+ TypeDispatch<x::IEWBinOp>(a, b, op);`
`68`	`68`	`}`