add missing test data.

Readme.md

@@ -1,5 +1,5 @@
 # netchdf
-_last updated: 7/18/2025_
+_last updated: 7/22/2025_
 
 This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. 
 
@@ -262,7 +262,7 @@
     return data as ArrayUByte. 
   * Netcdf-4 encodes CHAR values as HDF5 string type with elemSize = 1, so we use that convention to detect 
     legacy CHAR variables in HDF5 format. (NC_CHAR should not be used in new Netcdf-4 files, use NC_UBYTE or NC_STRING.) 
     Variables of type CHAR return data as STRING, since users can use UBYTE if thats what they intend.
   * Netcdf-4/HDF5 String variables may be fixed or variable length. For fixed Strings, we set the size of Datatype.STRING to 
     the fixed size. For both fixed and variable length Strings, the string will be truncated at the first zero byte, if any.
   * HDF4 does not have a STRING type, but does have signed and unsigned CHAR, and signed and unsigned BYTE. 

cli/build.gradle.kts

@@ -3,16 +3,16 @@
 }
 
 dependencies {
-    api(project(":core"))
+    implementation(project(":core"))
 
     implementation(libs.lzf)
     implementation(libs.lz4)
-    implementation(libs.kotlinx.cli)
+    // implementation(libs.kotlinx.cli)
     implementation(libs.oshai.logging)
     implementation(libs.logback.classic)
 
     testImplementation(kotlin("test"))
-    testImplementation(libs.junit.jupiter.params)
+    // testImplementation(libs.junit.jupiter.params)
 }
 
 kotlin {

cli/src/main/kotlin/com/sunya/netchdf/cli/ncdump.kt

@@ -1,8 +1,9 @@
 package com.sunya.netchdf.cli
 
-import com.sunya.netchdf.openNetchdfFile
-import kotlinx.cli.*
+// import com.sunya.netchdf.openNetchdfFile
+// import kotlinx.cli.*
 
+/*
 object ncdump {
     @JvmStatic
     fun main(args: Array<String>) {
@@ -24,4 +25,6 @@ object ncdump {
             println(myfile.cdl())
         }
     }
-}
+}
+
+ */

cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt

@@ -1,8 +1,8 @@
 package com.sunya.netchdf.hdf5
 
 import com.sunya.cdm.api.computeSize
-import kotlin.test.Test
 import com.sunya.netchdf.openNetchdfFile
+import kotlin.test.Test
 
 class TestFilters {
     init {

core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt

@@ -169,3 +169,17 @@ fun convertFromBytes(datatype : Datatype<*>, ba: ByteArray, isBE: Boolean, chars
         else -> throw IllegalArgumentException("datatype ${datatype}")
     }
 }
+
+
+fun Long.reverseByteOrder(wantBE: Boolean): Long {
+    val b1 = (this and 0xff).toByte()
+    val b2 = ((this ushr 8) and 0xff).toByte()
+    val b3 = ((this ushr 16) and 0xff).toByte()
+    val b4 = ((this ushr 24) and 0xff).toByte()
+    val b5 = ((this ushr 32) and 0xff).toByte()
+    val b6 = ((this ushr 40) and 0xff).toByte()
+    val b7 = ((this ushr 48) and 0xff).toByte()
+    val b8 = ((this ushr 56) and 0xff).toByte()
+
+    return convertToLong(byteArrayOf(b8, b7, b6, b5, b4, b3, b2, b1), 0, wantBE)
+}

core/src/commonMain/kotlin/com/sunya/cdm/util/IO.kt

@@ -30,15 +30,3 @@ internal fun IOcopyB(input: InputStream, out: OutputStream, bufferSize: Int): Lo
     out.flush()
     return totalBytesRead
 }
-
-/*
-fun Long.reverseByteOrder(): Long {
-    return ByteBuffer.allocate(Long.SIZE_BYTES)
-        .order(ByteOrder.LITTLE_ENDIAN)
-        .putLong(this)
-        .rewind()
-        .order(ByteOrder.BIG_ENDIAN)
-        .long
-}
-
- */

core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt

@@ -40,16 +40,6 @@ internal class FilterPipeline(
     val isBE: Boolean
 ) {
 
-    init {
-        if (mfp != null) {
-            mfp.filters.forEach { filter ->
-                if (filter.filterType == FilterType.lz4) {
-                    println("GOT LZ4!")
-                }
-            }
-        }
-    }
-
     fun apply(encodedData: ByteArray, filterMask: Int): ByteArray {
         if (mfp == null) return encodedData
         var data = encodedData

core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt

@@ -13,109 +13,13 @@ class JhdfReadTest {
         }
     }
 
-    @Test
-    fun problem() {
-        val filename = "$testData/jhdf/compound_datasets_earliest.hdf5"
-        println(filename)
-        readNetchdfData(filename, "vlen_chunked_compound", null, true, true)
-    }
-
-    // HDF5 "$testData/jhdf/bitshuffle_datasets.hdf5" {
-    //GROUP "/" {
-    //   DATASET "float32_bs0_comp0" {
-    //      DATATYPE  H5T_IEEE_F32LE
-    //      DATASPACE  SIMPLE { ( 20 ) / ( 20 ) }
-    //   }
-    //   DATASET "float32_bs0_comp2" {
-    //      DATATYPE  H5T_IEEE_F32LE
-    //      DATASPACE  SIMPLE { ( 20 ) / ( 20 ) }
-    //   } ...
-   //  @Test
-    fun testBitShuffle() { // 3
-        val filename = "$testData/jhdf/bitshuffle_datasets.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    @Test
-    fun testBTree2() { // 4
-        val filename = "$testData/jhdf/chunked_v4_datasets.hdf5 "
-        println(filename)
-        readNetchdfData(filename, "/filtered_btree_v2/int8", null, false, true)
-        readNetchdfData(filename, "/filtered_extensible_array/int8", null, false, true)
-        readNetchdfData(filename, "/filtered_extensible_array/large_int16", null, false, true)
-        readNetchdfData(filename, null, null, true, true)
-    }
-
-    // getHeapDataAsArray datatype=vlen null
-    @Test
-    fun testVlenHeapData() { // 6
-        val filename = "$testData/jhdf/compound_datasets_earliest.hdf5"
-        println(filename)
-        readNetchdfData(filename, "vlen_chunked_compound", null, true, false)
-    }
-
-    // see jhdf/src/main/java/io/jhdf/object/datatype/FloatingPoint.java
-    @Test
-    fun testHalfFloat() { // 12
-        val filename = "$testData/jhdf/float_special_values_earliest.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    // private typedefs
-    @Test
-    fun testCompoundTypedef() { // 18
-        val filename = "$testData/jhdf/isssue-523.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    @Test
-    fun testFractalHeap() { // 28
-        val filename = "$testData/jhdf/test_attribute_latest.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    // Unknown filter type= lzf name = lzf
-    // @Test
-    fun testLzfFilter() { // 37
-        val filename = "$testData/jhdf/test_compressed_chunked_datasets_earliest.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    // private typedefs
-    @Test
-    fun testVlen() { // 63
-        val filename = "$testData/jhdf/test_vlen_datasets_latest.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    @Test
-    fun testCompoundAttribute() {
-        val filename = "$testData/jhdf/test_compound_scalar_attribute.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
-    @Test
-    fun testCommittedDatatype() {
-        val filename = "$testData/jhdf/issue255_example.hdf5"
-        println(filename)
-        readNetchdfData(filename, null, null, true, false)
-    }
-
     /////////////////////////////////////
     @Test
     fun testReadAllJhdfFiles() {
         files().forEach { filename ->
             println(filename)
             readNetchdfData(filename, null, null, true, false)
         }
-
     }
 
 }

core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt

@@ -4,7 +4,7 @@ import okio.FileSystem
 import okio.Path
 import okio.Path.Companion.toPath
 
-const val testData = "src/commonTest/data/"
+const val testData = "../core/src/commonTest/data/"
 
 fun testFilesIn(dirPath: String): TestFiles.SequenceBuilder {
     return TestFiles.SequenceBuilder(dirPath)

testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5CTypeInfo.kt

@@ -182,6 +182,7 @@ internal data class H5CTypeInfo(val type_id: Long, val type_class : Int, val ele
                     1 -> Datatype.ENUM1.withTypedef(typedef)
                     2 -> Datatype.ENUM2.withTypedef(typedef)
                     4 -> Datatype.ENUM4.withTypedef(typedef)
+                    8 -> Datatype.ENUM8.withTypedef(typedef)
                     else -> throw RuntimeException("Bad hdf5 enum type with size= ${this.elemSize}")
                 }
             }

testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt

@@ -241,7 +241,10 @@ class H5Cbuilder(val filename: String) {
             val ltype = H5L_info2_t.`type$get`(linkinfo, 0L) // H5L_type_t
             if (ltype == H5L_TYPE_HARD()) {
                 val address = H5L_info2_t.u.ofAddress(linkinfo, context.arena)
-                if (debug) println("${indent}H5L_TYPE_HARD, address=$address")
+                if (debug) {
+                    val addressp = address.address()
+                    println("${indent}H5L_TYPE_HARD, address=$addressp reverseBE=${addressp.reverseByteOrder(true)} reverseLE=${addressp.reverseByteOrder(false)}")
+                }
             } else if (ltype == H5L_TYPE_SOFT()) {
                 val val_size = H5L_info2_t.u.`val_size$get`(linkinfo, 0L)
                 if (debug) println("${indent}H5L_TYPE_SOFT, val_size=$val_size")
@@ -278,6 +281,11 @@ class H5Cbuilder(val filename: String) {
             val loc_id = H5Oopen(group_id, linkname_p, H5P_DEFAULT_LONG)
 
             val oinfo_p = H5O_info2_t.allocate(context.arena)
+            val errno = H5Oget_info3(loc_id, oinfo_p, H5O_INFO_ALL())
+            if (errno < 0) {
+                println("H5Oget_info3 failed ($errno) on linkname $linkname")
+                return 0 // bogus "no error"
+            }
             checkErr("H5Oget_info3", H5Oget_info3(loc_id, oinfo_p, H5O_INFO_ALL()))
             val otype = H5O_info2_t.`type$get`(oinfo_p)
             val num_attr = H5O_info2_t.`num_attrs$get`(oinfo_p)
@@ -506,13 +514,9 @@ class H5Cbuilder(val filename: String) {
             if (isVariable) context.group.addVariable(vb)
 
             // datasetId is transient
-            var address = H5Dget_offset(datasetId)
-            /* println("**H5Cbuilder obj_name=$obj_name datasetId=$datasetId address=$address") // maybe there a byte order problem ??
-        if (address < 0) {
-            val reversed = datasetId.reverseByteOrder() // doesnt work
-            address = H5Dget_offset(reversed)
-            println("   try again with byte order reversed: datasetId=$reversed address=$address") // maybe there a byte order problem ??
-        } */
+            val address = H5Dget_offset(datasetId)
+            // println("**H5Cbuilder obj_name=$obj_name datasetId=$datasetId address=$address") // maybe there a byte order problem ??
+
             if (address > 0) datasetMap[address] = Pair(context.group, vb)
 
             if (obj_name.startsWith("StructMetadata")) {

testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt

@@ -69,8 +69,9 @@ fun compareGroup(group: Group, cgroup: Group, indent: Indent, showCompare: Boole
     }
 
     group.variables.forEach { v ->
-        val cv = cgroup.variables.find { it.name == v.name }!!
-        compareVariable(v, cv, indent.incr(), showCompare)
+        val cv = cgroup.variables.find { it.name == v.name }
+        if (cv == null) println("***cant find c variable ${v.fullname()}")
+        else compareVariable(v, cv, indent.incr(), showCompare)
     }
 
     group.groups.forEach { nested ->