LZ4 compressor + generalize reference file testing

.github/workflows/publish.yml

@@ -28,3 +28,7 @@ jobs:
         run: mvn --batch-mode deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+          OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+          MAVEN_GPG_KEY: ${{ secrets.MAVEN_GPG_KEY }}

.github/workflows/test.yml

@@ -31,9 +31,9 @@ jobs:
         with:
           python-version: '3.12'
 
-      - name: Install test file generator requirements
+      - name: Install reference file generator requirements
         run: |
-          pip install asdf numpy
+          pip install asdf numpy lz4
 
       - name: Locate Python
         id: locate-python

asdf-compression-commons-compress/pom.xml

@@ -10,7 +10,7 @@
     <artifactId>asdf-compression-commons-compress</artifactId>
 
     <name>asdf-compression-commons-compress</name>
-    <description>Support for BZip2 and LZ4 Frame compressed blocks</description>
+    <description>Support for BZip2 and LZ4 compressed blocks</description>
 
     <dependencies>
         <dependency>

asdf-compression-commons-compress/src/main/java/org/asdfformat/asdf/io/compression/Lz4Compressor.java

@@ -0,0 +1,46 @@
+package org.asdfformat.asdf.io.compression;
+
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
+import org.asdfformat.asdf.io.util.IOUtils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+public class Lz4Compressor implements Compressor {
+    public static final byte[] IDENTIFIER = {108, 122, 52, 0}; // 'lz4' + padding
+
+    @Override
+    public byte[] getIdentifier() {
+        return IDENTIFIER;
+    }
+
+    @Override
+    public long decompress(final ByteBuffer inputBuffer, final ByteBuffer outputBuffer) throws IOException {
+        long bytesDecompressed = 0L;
+
+        while (inputBuffer.hasRemaining()) {
+            inputBuffer.order(ByteOrder.BIG_ENDIAN);
+            final int lz4BlockLength = inputBuffer.getInt() - 4;
+            if (lz4BlockLength < 0) {
+                throw new RuntimeException("LZ4 block length > " + Integer.MAX_VALUE + " not supported");
+            }
+
+            // Discard the uncompressed data size written by the Python LZ4 bindings:
+            inputBuffer.getInt();
+
+            final ByteBuffer lz4BlockInputBuffer = inputBuffer.duplicate();
+            lz4BlockInputBuffer.limit(inputBuffer.position() + lz4BlockLength);
+
+            try (final ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(lz4BlockInputBuffer);
+                 final BlockLZ4CompressorInputStream blockLZ4CompressorInputStream = new BlockLZ4CompressorInputStream(byteBufferInputStream)
+            ) {
+                bytesDecompressed += IOUtils.transferTo(blockLZ4CompressorInputStream, outputBuffer);
+            }
+
+            inputBuffer.position(inputBuffer.position() + lz4BlockLength);
+        }
+
+        return bytesDecompressed;
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/Lz4CompressorReferenceTest.java

@@ -0,0 +1,35 @@
+package org.asdfformat.asdf.io.compression;
+
+import org.asdfformat.asdf.Asdf;
+import org.asdfformat.asdf.AsdfFile;
+import org.asdfformat.asdf.io.compression.testing.CommonsCompressReferenceFileType;
+import org.asdfformat.asdf.ndarray.DoubleNdArray;
+import org.asdfformat.asdf.standard.AsdfStandardType;
+import org.asdfformat.asdf.testing.ReferenceFileUtils;
+import org.junit.jupiter.api.Tag;
+import org.junitpioneer.jupiter.cartesian.CartesianTest;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.asdfformat.asdf.io.compression.testing.TestCategories.REFERENCE_TESTS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag(REFERENCE_TESTS)
+public class Lz4CompressorReferenceTest {
+    @CartesianTest
+    public void test1d(
+            @CartesianTest.Enum(value = CommonsCompressReferenceFileType.class, names = {"NDARRAY_COMPRESSED_LZ4"}) final CommonsCompressReferenceFileType referenceFileType,
+            @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType
+    ) throws IOException {
+        final Path path = ReferenceFileUtils.getPath(referenceFileType, asdfStandardType.getVersion());
+
+        try (final AsdfFile asdfFile = Asdf.open(path)) {
+            final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray();
+
+            for (int i = 0; i < 10; i++) {
+                assertEquals(doubleNdArray.get(i), i);
+            }
+        }
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/CommonsCompressReferenceFileType.java

@@ -0,0 +1,23 @@
+package org.asdfformat.asdf.io.compression.testing;
+
+import org.asdfformat.asdf.testing.ReferenceFile;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public enum CommonsCompressReferenceFileType implements ReferenceFile {
+    NDARRAY_COMPRESSED_LZ4,
+    ;
+
+    @Override
+    public String getName() {
+        return name();
+    }
+
+    @Override
+    public InputStream openScript() throws IOException {
+        return CommonsCompressReferenceFileType.class.getResourceAsStream(
+                String.format("/test-file-scripts/%s.py", name().toLowerCase())
+        );
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/TestCategories.java

@@ -0,0 +1,5 @@
+package org.asdfformat.asdf.io.compression.testing;
+
+public class TestCategories {
+    public static final String REFERENCE_TESTS = "reference-tests";
+}

asdf-compression-commons-compress/src/test/resources/test-file-scripts/ndarray_compressed_lz4.py

@@ -0,0 +1,3 @@
+af["arr"] = np.arange(0, 10, dtype=np.float64)
+
+af.set_array_compression(af["arr"], "lz4")

asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java

@@ -22,6 +22,11 @@ public class Compressors {
                     .identifier("bzp2".getBytes(StandardCharsets.UTF_8))
                     .moduleName("asdf-compression-commons-compress")
                     .build(),
+            OptionalCompressor.builder()
+                    .className("org.asdfformat.asdf.io.compression.Lz4Compressor")
+                    .identifier(new byte[] {108, 122, 52, 0})
+                    .moduleName("asdf-compression-commons-compress")
+                    .build(),
             OptionalCompressor.builder()
                     .className("org.asdfformat.asdf.io.compression.Lz4FrameCompressor")
                     .identifier("lz4f".getBytes(StandardCharsets.UTF_8))

asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFile.java

@@ -0,0 +1,9 @@
+package org.asdfformat.asdf.testing;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface ReferenceFile {
+    String getName();
+    InputStream openScript() throws IOException;
+}

asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFiles.java → asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFileUtils.java

@@ -15,7 +15,7 @@
 import java.util.Map;
 import java.util.Optional;
 
-public class TestFiles {
+public class ReferenceFileUtils {
     private static final String PYTHON_PATH = System.getenv("ASDF_JAVA_TESTS_PYTHON_PATH");
     private static final Path TEST_FILE_GENERATOR_PY_PATH = getTestFileGeneratorPyPath();
 
@@ -26,8 +26,8 @@ private static Path getTestFileGeneratorPyPath() {
         final Path path = file.toPath();
 
         try (
-                final InputStream inputStream = Optional.ofNullable(TestFiles.class.getResourceAsStream("/generation/test_file_generator.py"))
-                        .orElseThrow(() -> new RuntimeException("Missing generation/test_file_generator.py"));
+                final InputStream inputStream = Optional.ofNullable(ReferenceFileUtils.class.getResourceAsStream("/testing/reference_file_generator.py"))
+                        .orElseThrow(() -> new RuntimeException("Missing testing/reference_file_generator.py"));
                 final OutputStream outputStream = Files.newOutputStream(path, StandardOpenOption.CREATE)
         ) {
             IOUtils.transferTo(inputStream, outputStream);
@@ -38,24 +38,24 @@ private static Path getTestFileGeneratorPyPath() {
 
     private static final Map<String, Path> TEST_FILES = new HashMap<>();
 
-    public static Path getPath(final TestFileType testFileType, final Version asdfStandardVersion) {
-        final String key = makeKey(testFileType, asdfStandardVersion);
+    public static Path getPath(final ReferenceFile testfile, final Version asdfStandardVersion) {
+        final String key = makeKey(testfile, asdfStandardVersion);
 
         if (!TEST_FILES.containsKey(key)) {
-            TEST_FILES.put(key, generateTestFile(testFileType, asdfStandardVersion));
+            TEST_FILES.put(key, generateTestFile(testfile, asdfStandardVersion));
         }
 
         return TEST_FILES.get(key);
     }
 
     @SneakyThrows(IOException.class)
-    private static Path generateTestFile(final TestFileType testFileType, final Version asdfStandardVersion) {
-        try (final InputStream scriptInputStream = TestFiles.class.getResourceAsStream(testFileType.getScriptResourceName())) {
+    private static Path generateTestFile(final ReferenceFile referenceFile, final Version asdfStandardVersion) {
+        try (final InputStream scriptInputStream = referenceFile.openScript()) {
             if (scriptInputStream == null) {
-                throw new RuntimeException("Missing generator script for " + testFileType);
+                throw new RuntimeException("Missing generator script for " + referenceFile.getName());
             }
 
-            final File file = File.createTempFile(testFileType + "-", ".asdf");
+            final File file = File.createTempFile(referenceFile.getName() + "-", ".asdf");
             file.deleteOnExit();
             final Path path = file.toPath();
 
@@ -75,7 +75,7 @@ private static Path generateTestFile(final TestFileType testFileType, final Vers
         }
     }
 
-    private static String makeKey(final TestFileType testFileType, final Version asdfStandardVersion) {
-        return String.format("%s:%s", testFileType, asdfStandardVersion);
+    private static String makeKey(final ReferenceFile referenceFile, final Version asdfStandardVersion) {
+        return String.format("%s:%s", referenceFile, asdfStandardVersion);
     }
 }

asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayCompressedZlibReferenceTest.java

@@ -3,8 +3,8 @@
 import org.asdfformat.asdf.Asdf;
 import org.asdfformat.asdf.AsdfFile;
 import org.asdfformat.asdf.standard.AsdfStandardType;
-import org.asdfformat.asdf.testing.TestFileType;
-import org.asdfformat.asdf.testing.TestFiles;
+import org.asdfformat.asdf.testing.CoreReferenceFileType;
+import org.asdfformat.asdf.testing.ReferenceFileUtils;
 import org.junit.jupiter.api.Tag;
 import org.junitpioneer.jupiter.cartesian.CartesianTest;
 
@@ -18,10 +18,10 @@
 public class NdArrayCompressedZlibReferenceTest {
     @CartesianTest
     public void test1d(
-            @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_COMPRESSED_ZLIB"}) final TestFileType testFileType,
+            @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_COMPRESSED_ZLIB"}) final CoreReferenceFileType coreTestFileType,
             @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType
     ) throws IOException {
-        final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion());
+        final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion());
 
         try (final AsdfFile asdfFile = Asdf.open(path)) {
             final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray();

asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayFloat64ReferenceTest.java

@@ -3,8 +3,8 @@
 import org.asdfformat.asdf.Asdf;
 import org.asdfformat.asdf.AsdfFile;
 import org.asdfformat.asdf.standard.AsdfStandardType;
-import org.asdfformat.asdf.testing.TestFileType;
-import org.asdfformat.asdf.testing.TestFiles;
+import org.asdfformat.asdf.testing.CoreReferenceFileType;
+import org.asdfformat.asdf.testing.ReferenceFileUtils;
 import org.junit.jupiter.api.Tag;
 import org.junitpioneer.jupiter.cartesian.CartesianTest;
 
@@ -18,10 +18,10 @@
 public class NdArrayFloat64ReferenceTest {
     @CartesianTest
     public void test1d(
-            @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_FLOAT64_1D_BLOCK_BIG", "NDARRAY_FLOAT64_1D_BLOCK_LITTLE", "NDARRAY_FLOAT64_1D_INLINE"}) final TestFileType testFileType,
+            @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_FLOAT64_1D_BLOCK_BIG", "NDARRAY_FLOAT64_1D_BLOCK_LITTLE", "NDARRAY_FLOAT64_1D_INLINE"}) final CoreReferenceFileType coreTestFileType,
             @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType
     ) throws IOException {
-        final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion());
+        final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion());
 
         try (final AsdfFile asdfFile = Asdf.open(path)) {
             final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray();

asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayStructuredReferenceTest.java

@@ -3,8 +3,8 @@
 import org.asdfformat.asdf.Asdf;
 import org.asdfformat.asdf.AsdfFile;
 import org.asdfformat.asdf.standard.AsdfStandardType;
-import org.asdfformat.asdf.testing.TestFileType;
-import org.asdfformat.asdf.testing.TestFiles;
+import org.asdfformat.asdf.testing.CoreReferenceFileType;
+import org.asdfformat.asdf.testing.ReferenceFileUtils;
 import org.junit.jupiter.api.Tag;
 import org.junitpioneer.jupiter.cartesian.CartesianTest;
 
@@ -18,10 +18,10 @@
 public class NdArrayStructuredReferenceTest {
     @CartesianTest
     public void test1d(
-            @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_STRUCTURED_1D_BLOCK"}) final TestFileType testFileType,
+            @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_STRUCTURED_1D_BLOCK"}) final CoreReferenceFileType coreTestFileType,
             @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType
     ) throws IOException {
-        final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion());
+        final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion());
 
         try (final AsdfFile asdfFile = Asdf.open(path)) {
             final TupleNdArray tupleNdArray = asdfFile.getTree().getNdArray("arr").asTupleNdArray();

asdf-core/src/test/java/org/asdfformat/asdf/testing/CoreReferenceFileType.java

@@ -0,0 +1,25 @@
+package org.asdfformat.asdf.testing;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public enum CoreReferenceFileType implements ReferenceFile {
+    NDARRAY_COMPRESSED_ZLIB,
+    NDARRAY_FLOAT64_1D_BLOCK_BIG,
+    NDARRAY_FLOAT64_1D_BLOCK_LITTLE,
+    NDARRAY_FLOAT64_1D_INLINE,
+    NDARRAY_STRUCTURED_1D_BLOCK,
+    ;
+
+    @Override
+    public String getName() {
+        return name();
+    }
+
+    @Override
+    public InputStream openScript() throws IOException {
+        return CoreReferenceFileType.class.getResourceAsStream(
+                String.format("/reference-file-scripts/%s.py", name().toLowerCase())
+        );
+    }
+}

pom.xml

@@ -35,7 +35,7 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <maven.compiler.release>8</maven.compiler.release>
-    <revision>0.1-alpha-3</revision>
+    <revision>0.1-alpha-6</revision>
     <lombok.version>1.18.36</lombok.version>
     <snakeyaml.version>2.4</snakeyaml.version>
     <maven-clean-plugin.version>3.4.0</maven-clean-plugin.version>
@@ -49,7 +49,7 @@
     <maven-deploy-plugin.version>3.1.4</maven-deploy-plugin.version>
     <maven-source-plugin.version>2.2.1</maven-source-plugin.version>
     <maven-javadoc-plugin.version>3.11.2</maven-javadoc-plugin.version>
-    <maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
+    <maven-gpg-plugin.version>3.2.4</maven-gpg-plugin.version>
     <maven-surefire-plugin.version>3.5.3</maven-surefire-plugin.version>
     <central-publishing-plugin.version>0.7.0</central-publishing-plugin.version>
     <junit.version>5.11.4</junit.version>
@@ -187,6 +187,9 @@
               <goals>
                 <goal>sign</goal>
               </goals>
+              <configuration>
+                <signer>bc</signer>
+              </configuration>
             </execution>
           </executions>
         </plugin>