fixing reshape communication (#26)

* fixing reshape communication. Together with new imex tests now pass with 3 and for ranks
* workaround for long filepaths in CI

Author: fschlimb

.github/workflows/ci.yml

@@ -122,17 +122,24 @@ jobs:
           . $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
           . $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
           MLIRROOT=$GITHUB_WORKSPACE/third_party/install/llvm-mlir IMEXROOT=$GITHUB_WORKSPACE/third_party/install/imex python setup.py install
-          DDPT_ROOT=`pip show -f ddptensor | grep Location | awk '{print $2}'`
-          echo "DDPT_IDTR_SO=${DDPT_ROOT}/ddptensor/libidtr.so" >> $GITHUB_ENV
+          # DDPT_ROOT=`pip show -f ddptensor | grep Location | awk '{print $2}'`
+          # for some reason we cannot use the idtr from the install, not even a copy works
+          libidtr=`ls ${PWD}/build/lib.linux-x86_64-cpython-*/ddptensor/libidtr.so`
+          echo "DDPT_IDTR_SO=${libidtr}" >> $GITHUB_ENV
           echo "DDPT_CRUNNER_SO=${GITHUB_WORKSPACE}/third_party/install/llvm-mlir/lib/libmlir_c_runner_utils.so" >> $GITHUB_ENV
+          echo $DDPT_IDTR_SO
       - name: Test DDPT (pytest)
         run: |
           . $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
           . $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
           cd test
-          pytest .
-          DDPT_FORCE_DIST=1 pytest .
-          mpirun -n 2 pytest .
+          echo $DDPT_IDTR_SO
+          ldd $DDPT_IDTR_SO
+          pytest . -v -s
+          DDPT_FORCE_DIST=1 pytest . -v -s
+          mpirun -n 2 pytest . -v -s
+          mpirun -n 3 pytest . -v -s
+          mpirun -n 4 pytest . -v -s
           cd -
       - name: Run examples
         run: |

imex_version.txt

@@ -1 +1 @@
-2726c17193e3297a5442ba507ba560133723ac09
+c86b38a055b73c9041ca38dd36760fa11d3d5b9f

src/idtr.cpp

@@ -224,7 +224,7 @@ void copy_(uint64_t d, uint64_t &pos, T *cptr, const int64_t *sizes,
   uint64_t first = 0;
   if (pos < start) {
     first = (start - pos) / chunk;
-    pos += first;
+    pos += first * chunk;
     cptr += first * stride;
     assert(pos <= start && pos < end);
   }
@@ -281,9 +281,11 @@ template <typename T>
 void _idtr_reduce_all(int64_t dataRank, void *dataDescr, int op) {
   UnrankedMemRefType<T> data(dataRank, dataDescr);
   assert(dataRank == 0 || (dataRank == 1 && data.strides()[0] == 1));
-  getTransceiver()->reduce_all(
-      data.data(), DTYPE<T>::value, dataRank ? data.sizes()[0] : 1,
-      mlir2ddpt(static_cast<imex::ptensor::ReduceOpId>(op)));
+  auto d = data.data();
+  auto t = DTYPE<T>::value;
+  auto r = dataRank ? data.sizes()[0] : 1;
+  auto o = mlir2ddpt(static_cast<imex::ptensor::ReduceOpId>(op));
+  getTransceiver()->reduce_all(d, t, r, o);
 }
 
 extern "C" {