seperate chunk iterator tests out

core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt

@@ -124,164 +124,3 @@ fun readMiddleSection(myfile: Netchdf, myvar: Variable<*>, shape: LongArray, sho
     if (showData) println(mydata)
 }
 
-//////////////////////////////////////////////////////////////////////////////////////
-// compare reading data regular and through the chunkIterate API
-
-fun compareNetchIterate(filename: String, varname : String? = null, compare : Boolean = true) {
-    openNetchdfFile(filename).use { myfile ->
-        if (myfile == null) {
-            println("*** not a netchdf file = $filename")
-            return
-        }
-        println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes")
-        var countChunks = 0
-        if (varname != null) {
-            val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname")
-            countChunks +=  compareOneVarIterate(myfile, myvar, compare)
-        } else {
-            myfile.rootGroup().allVariables().forEach { it ->
-                countChunks += compareOneVarIterate(myfile, it, compare)
-            }
-        }
-        if (countChunks > 0) {
-            println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks")
-        }
-    }
-}
-
-// compare readArrayData with chunkIterator
-fun compareOneVarIterate(myFile: Netchdf, myvar: Variable<*>, compare : Boolean = true) : Int {
-    val filename = myFile.location().substringAfterLast('/')
-    val varBytes = myvar.nelems
-    if (varBytes >= maxBytes) {
-        println(" *** ${myvar.nameAndShape()} skip reading ArrayData too many bytes= $varBytes max = $maxBytes")
-        return 0
-    }
-
-    val sum1 = AtomicDouble(0.0)
-    val sumArrayData = if (compare) {
-        val time3 = measureNanoTime {
-            val arrayData = myFile.readArrayData(myvar, null)
-            sumValues(arrayData, sum1)
-        }
-        Stats.of("readArrayData", filename, "chunk").accum(time3, 1)
-        sum1.get()
-    } else 0.0
-
-    val sum2 = AtomicDouble(0.0)
-    var countChunks = 0
-    val time1 = measureNanoTime {
-        val chunkIter = myFile.chunkIterator(myvar)
-        for (pair in chunkIter) {
-            // println(" ${pair.section} = ${pair.array.shape.contentToString()}")
-            sumValues(pair.array, sum2)
-            countChunks++
-        }
-    }
-    val sumChunkIterator = sum2.get()
-    if (compare) Stats.of("chunkIterator", filename, "chunk").accum(time1, countChunks)
-
-    if (compare && sumChunkIterator.isFinite() && sumArrayData.isFinite()) {
-        // println("  sumChunkIterator = $sumChunkIterator for ${myvar.nameAndShape()}")
-        assertTrue(nearlyEquals(sumArrayData, sumChunkIterator), "chunkIterator $sumChunkIterator != $sumArrayData sumArrayData")
-    }
-    return countChunks
-}
-
-//////////////////////////////////////////////////////////////////////////////////////
-// compare reading data chunkIterate API with Netch and NC
-
-fun compareIterateWithNC(myfile: Netchdf, ncfile: Netchdf, varname: String?, section: SectionPartial? = null) {
-    if (varname != null) {
-        val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname }
-        if (myvar == null) {
-            println(" *** cant find myvar $varname")
-            return
-        }
-        val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() }
-        if (ncvar == null) {
-            throw RuntimeException(" *** cant find ncvar $varname")
-        }
-        compareOneVarIterate(myvar, myfile, ncvar, ncfile, section)
-    } else {
-        myfile.rootGroup().allVariables().forEach { myvar ->
-            val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() }
-            if (ncvar == null) {
-                println(" *** cant find ${myvar.fullname()} in ncfile")
-            } else {
-                compareOneVarIterate(myvar, myfile, ncvar, ncfile, null)
-            }
-        }
-    }
-}
-
-fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, ncvar : Variable<*>, ncfile: Netchdf, section: SectionPartial?) {
-    val sum = AtomicDouble(0.0)
-    var countChunks = 0
-    val time1 = measureNanoTime {
-        val chunkIter = myfile.chunkIterator(myvar)
-        for (pair in chunkIter) {
-            // println(" ${pair.section} = ${pair.array.shape.contentToString()}")
-            sumValues(pair.array, sum)
-            countChunks++
-        }
-    }
-    Stats.of("netchdf", myfile.location(), "chunk").accum(time1, countChunks)
-    val sum1 = sum.get()
-
-    sum.set(0.0)
-    countChunks = 0
-    val time2 = measureNanoTime {
-        val chunkIter = ncfile.chunkIterator(ncvar)
-        for (pair in chunkIter) {
-            // println(" ${pair.section} = ${pair.array.shape.contentToString()}")
-            sumValues(pair.array, sum)
-            countChunks++
-        }
-    }
-    Stats.of("nclib", ncfile.location(), "chunk").accum(time2, countChunks)
-    val sum2 = sum.get()
-
-    if (sum1.isFinite() && sum2.isFinite()) {
-        assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2")
-        println("sum = $sum1")
-    }
-}
-
-///////////////////////////////////////////////////////////
-fun sumValues(array : ArrayTyped<*>, sum : AtomicDouble) {
-    if (array is ArraySingle || array is ArrayEmpty) {
-        return // test fillValue the same ??
-    }
-
-    if (array.datatype.isNumber) {
-        for (value in array) {
-            val number = (value as Number)
-            val numberd: Double = number.toDouble()
-            if (numberd.isFinite()) {
-                sum.getAndAdd(numberd)
-            }
-        }
-    } else if (array.datatype.isIntegral) {
-        for (value in array) {
-            val useValue = when (value) {
-                is UByte -> value.toByte()
-                is UShort -> value.toShort()
-                is UInt -> value.toInt()
-                is ULong -> value.toLong()
-                else -> value
-            }
-            val number = (useValue as Number)
-            val numberd: Double = number.toDouble()
-            if (numberd.isFinite()) {
-                sum.getAndAdd(numberd)
-            }
-        }
-    }
-}
-
-fun measureNanoTime (block: () -> Unit): Long {
-    return 0
-}
-
-

testfiles/src/test/kotlin/com/sunya/netchdf/NetchdfTest.kt

@@ -3,7 +3,6 @@ package com.sunya.netchdf
 import com.sunya.cdm.api.*
 import com.sunya.netchdf.testfiles.*
 import com.sunya.netchdf.testutils.Stats
-import com.sunya.netchdf.testutils.compareNetchIterate
 import com.sunya.netchdf.testutils.readNetchdfData
 import com.sunya.netchdf.testutils.showNetchdfHeader
 import com.sunya.netchdf.testutils.testData
@@ -125,9 +124,9 @@ class NetchdfTest {
         readNetchdfData(filename)
     }
 
-    // TODO too slow
+    /* TODO too slow
     // @Test
     fun testReadNetchIterate(filename: String) {
         compareNetchIterate(filename)
-    }
+    } */
 }

testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt

@@ -6,7 +6,6 @@ import com.sunya.netchdf.openNetchdfFile
 import com.sunya.netchdf.openNetchdfFileWithFormat
 import com.sunya.netchdf.testfiles.H4Files
 import com.sunya.netchdf.testutils.Stats
-import com.sunya.netchdf.testutils.compareNetchIterate
 import com.sunya.netchdf.testutils.readNetchdfData
 import com.sunya.netchdf.testutils.testData
 import org.junit.jupiter.params.ParameterizedTest
@@ -121,10 +120,10 @@ class H4readTest {
         }
     }
 
-    // takes too long : 47 hours with, 2 hours without (luckily get to divide by 24 for wall clock)
+    /* takes too long : 47 hours with, 2 hours without (luckily get to divide by 24 for wall clock)
     // @Test
     fun testReadIterate(filename: String) {
         compareNetchIterate(filename)
-    }
+    } */
 
 }

testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt

@@ -0,0 +1,112 @@
+package com.sunya.netchdf.hdf5
+
+import com.sunya.cdm.api.Netchdf
+import com.sunya.cdm.api.Variable
+import com.sunya.cdm.api.chunkConcurrent
+import com.sunya.cdm.array.ArrayTyped
+import com.sunya.netchdf.testfiles.H5Files
+import com.sunya.netchdf.testutils.AtomicDouble
+import com.sunya.netchdf.testutils.Stats
+import com.sunya.netchdf.testutils.compareNetchIterate
+import com.sunya.netchdf.testutils.measureNanoTime
+import kotlin.collections.iterator
+
+import kotlin.test.*
+
+// Sanity check read Hdf5File header, for non-netcdf4 files
+class H5readConcurrentTest {
+
+    companion object {
+        @JvmStatic
+        fun files(): Iterator<String> {
+            return H5Files.files()
+        }
+    }
+
+    @Test
+    fun testReadIterate() {
+        files().forEach { filename ->
+            compareNetchIterate(filename, null)
+        }
+    }
+
+    @Test
+    fun testReadConcurrent() {
+        files().forEach { filename ->
+            readH5concurrent(filename, null)
+        }
+    }
+}
+
+
+fun readH5concurrent(filename: String, varname: String? = null) {
+    Hdf5File(filename).use { myfile ->
+        println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes")
+        var countChunks = 0
+        if (varname != null) {
+            val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname }
+                ?: throw RuntimeException("cant find $varname")
+            countChunks += testOneVarConcurrent(myfile, myvar)
+        } else {
+            myfile.rootGroup().allVariables().forEach { it ->
+                if (it.datatype.isNumber) {
+                    countChunks += testOneVarConcurrent(myfile, it)
+                }
+            }
+        }
+        if (countChunks > 0) {
+            println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks")
+        }
+    }
+}
+
+fun testOneVarConcurrent(myFile: Netchdf, myvar: Variable<*>): Int {
+    val filename = myFile.location().substringAfterLast('/')
+    val sum = AtomicDouble(0.0)
+    var countChunks = 0
+    val time1 = measureNanoTime {
+        val chunkIter = myFile.chunkIterator(myvar)
+        for (pair in chunkIter) {
+            // println(" ${pair.section} = ${pair.array.shape.contentToString()}")
+            sumValues(pair.array)
+            countChunks++
+        }
+    }
+    val sum1 = sum.get()
+    Stats.of("serialSum", filename, "chunk").accum(time1, countChunks)
+
+    sum.set(0.0)
+    val time2 = measureNanoTime {
+        myFile.chunkConcurrent(myvar, null) { sumValues(it.array) }
+    }
+    val sum2 = sum.get()
+    Stats.of("concurrentSum", filename, "chunk").accum(time2, countChunks)
+
+    sum.set(0.0)
+    val time3 = measureNanoTime {
+        val arrayData = myFile.readArrayData(myvar, null)
+        sumValues(arrayData)
+    }
+    val sum3 = sum.get()
+    Stats.of("regularSum", filename, "chunk").accum(time3, countChunks)
+
+    /* if (sum1.isFinite() && sum2.isFinite() && sum3.isFinite()) {
+    assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2")
+    assertTrue(nearlyEquals(sum1, sum3), "$sum1 != $sum3 sum3")
+}
+
+ */
+    return countChunks
+}
+
+var sum = AtomicDouble(0.0)
+fun sumValues(array: ArrayTyped<*>) {
+    if (!array.datatype.isNumber or true) return
+    for (value in array) {
+        val number = (value as Number)
+        val numberd: Double = number.toDouble()
+        if (numberd.isFinite()) {
+            sum.getAndAdd(numberd)
+        }
+    }
+}