Fix key ordering in cbor maps

Improve error handling and reporting
Fix refs to read stream of objects
Restrict block put to a single file
Add more tests

Author: ianopolous

src/main/java/io/ipfs/api/IPFS.java

@@ -110,8 +110,12 @@ public InputStream catStream(Multihash hash) throws IOException {
         return retrieveStream("cat/" + hash);
     }
 
-    public Map refs(Multihash hash, boolean recursive) throws IOException {
-        return retrieveMap("refs?arg=" + hash +"&r="+recursive);
+    public List<Multihash> refs(Multihash hash, boolean recursive) throws IOException {
+        String jsonStream = new String(retrieve("refs?arg=" + hash + "&r=" + recursive));
+        return JSONParser.parseStream(jsonStream).stream()
+                .map(m -> (String) (((Map) m).get("Ref")))
+                .map(Cid::decode)
+                .collect(Collectors.toList());
     }
 
     public Map resolve(String scheme, Multihash hash, boolean recursive) throws IOException {
@@ -150,7 +154,7 @@ public class Pin {
         public List<Multihash> add(Multihash hash) throws IOException {
             return ((List<Object>)((Map)retrieveAndParse("pin/add?stream-channels=true&arg=" + hash)).get("Pins"))
                     .stream()
-                    .map(x -> Cid.decode((String)x))
+                    .map(x -> Cid.decode((String) x))
                     .collect(Collectors.toList());
         }
 
@@ -194,12 +198,16 @@ public List<MerkleNode> put(List<byte[]> data) throws IOException {
         }
 
         public List<MerkleNode> put(List<byte[]> data, Optional<String> format) throws IOException {
+            // N.B. Once IPFS implements a bulk put this can become a single multipart call with multiple 'files'
+            return data.stream().map(array -> put(array, format)).collect(Collectors.toList());
+        }
+
+        public MerkleNode put(byte[] data, Optional<String> format) {
             String fmt = format.map(f -> "&format=" + f).orElse("");
             Multipart m = new Multipart("http://" + host + ":" + port + version+"block/put?stream-channels=true" + fmt, "UTF-8");
-            for (byte[] f : data)
-                m.addFilePart("file", new NamedStreamable.ByteArrayWrapper(f));
+            m.addFilePart("file", new NamedStreamable.ByteArrayWrapper(data));
             String res = m.finish();
-            return JSONParser.parseStream(res).stream().map(x -> MerkleNode.fromJSON((Map<String, Object>) x)).collect(Collectors.toList());
+            return JSONParser.parseStream(res).stream().map(x -> MerkleNode.fromJSON((Map<String, Object>) x)).findFirst().get();
         }
 
         public Map stat(Multihash hash) throws IOException {
@@ -506,7 +514,8 @@ private static byte[] get(URL target) throws IOException {
         } catch (ConnectException e) {
             throw new RuntimeException("Couldn't connect to IPFS daemon at "+target+"\n Is IPFS running?");
         } catch (IOException e) {
-            throw new RuntimeException("IOException contacting IPFS daemon.\nTrailer: " + conn.getHeaderFields().get("Trailer"), e);
+            String err = new String(readFully(conn.getErrorStream()));
+            throw new RuntimeException("IOException contacting IPFS daemon.\nTrailer: " + conn.getHeaderFields().get("Trailer") + " " + err, e);
         }
     }
 
@@ -541,6 +550,10 @@ private static byte[] post(URL target, byte[] body, Map<String, String> headers)
         out.close();
 
         InputStream in = conn.getInputStream();
+        return readFully(in);
+    }
+
+    private static final byte[] readFully(InputStream in) throws IOException {
         ByteArrayOutputStream resp = new ByteArrayOutputStream();
         byte[] buf = new byte[4096];
         int r;

src/main/java/io/ipfs/api/Multipart.java

@@ -12,20 +12,24 @@ public class Multipart {
     private OutputStream out;
     private PrintWriter writer;
 
-    public Multipart(String requestURL, String charset) throws IOException {
+    public Multipart(String requestURL, String charset) {
         this.charset = charset;
 
         boundary = createBoundary();
 
-        URL url = new URL(requestURL);
-        httpConn = (HttpURLConnection) url.openConnection();
-        httpConn.setUseCaches(false);
-        httpConn.setDoOutput(true);
-        httpConn.setDoInput(true);
-        httpConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
-        httpConn.setRequestProperty("User-Agent", "Java IPFS CLient");
-        out = httpConn.getOutputStream();
-        writer = new PrintWriter(new OutputStreamWriter(out, charset), true);
+        try {
+            URL url = new URL(requestURL);
+            httpConn = (HttpURLConnection) url.openConnection();
+            httpConn.setUseCaches(false);
+            httpConn.setDoOutput(true);
+            httpConn.setDoInput(true);
+            httpConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
+            httpConn.setRequestProperty("User-Agent", "Java IPFS CLient");
+            out = httpConn.getOutputStream();
+            writer = new PrintWriter(new OutputStreamWriter(out, charset), true);
+        } catch (IOException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
     }
 
     public static String createBoundary() {
@@ -73,25 +77,29 @@ public void addDirectoryPart(String path) {
         }
     }
 
-    public void addFilePart(String fieldName, NamedStreamable uploadFile) throws IOException {
+    public void addFilePart(String fieldName, NamedStreamable uploadFile) {
         Optional<String> fileName = uploadFile.getName();
         writer.append("--" + boundary).append(LINE_FEED);
         if (!fileName.isPresent())
             writer.append("Content-Disposition: file; name=\"" + fieldName + "\";").append(LINE_FEED);
         else
-            writer.append("Content-Disposition: file; filename=\"" + fileName.get() + "\"").append(LINE_FEED);
+            writer.append("Content-Disposition: file; filename=\"" + fileName.get() + "\";").append(LINE_FEED);
         writer.append("Content-Type: application/octet-stream").append(LINE_FEED);
         writer.append("Content-Transfer-Encoding: binary").append(LINE_FEED);
         writer.append(LINE_FEED);
         writer.flush();
 
-        InputStream inputStream = uploadFile.getInputStream();
-        byte[] buffer = new byte[4096];
-        int r;
-        while ((r = inputStream.read(buffer)) != -1)
-            out.write(buffer, 0, r);
-        out.flush();
-        inputStream.close();
+        try {
+            InputStream inputStream = uploadFile.getInputStream();
+            byte[] buffer = new byte[4096];
+            int r;
+            while ((r = inputStream.read(buffer)) != -1)
+                out.write(buffer, 0, r);
+            out.flush();
+            inputStream.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
 
         writer.append(LINE_FEED);
         writer.flush();
@@ -102,35 +110,40 @@ public void addHeaderField(String name, String value) {
         writer.flush();
     }
 
-    public String finish() throws IOException {
+    public String finish() {
         StringBuilder b = new StringBuilder();
 
         writer.append("--" + boundary + "--").append(LINE_FEED);
         writer.close();
 
-        int status = httpConn.getResponseCode();
-        if (status == HttpURLConnection.HTTP_OK) {
-            BufferedReader reader = new BufferedReader(new InputStreamReader(
-                    httpConn.getInputStream()));
-            String line;
-            while ((line = reader.readLine()) != null) {
-                b.append(line);
-            }
-            reader.close();
-            httpConn.disconnect();
-        } else {
-            try {
+        try {
+            int status = httpConn.getResponseCode();
+            if (status == HttpURLConnection.HTTP_OK) {
                 BufferedReader reader = new BufferedReader(new InputStreamReader(
                         httpConn.getInputStream()));
                 String line;
                 while ((line = reader.readLine()) != null) {
                     b.append(line);
                 }
                 reader.close();
-            } catch (Throwable t) {}
-            throw new IOException("Server returned status: " + status + " with body: "+b.toString() + " and Trailer header: "+httpConn.getHeaderFields().get("Trailer"));
-        }
+                httpConn.disconnect();
+            } else {
+                try {
+                    BufferedReader reader = new BufferedReader(new InputStreamReader(
+                            httpConn.getInputStream()));
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        b.append(line);
+                    }
+                    reader.close();
+                } catch (Throwable t) {
+                }
+                throw new IOException("Server returned status: " + status + " with body: " + b.toString() + " and Trailer header: " + httpConn.getHeaderFields().get("Trailer"));
+            }
 
-        return b.toString();
+            return b.toString();
+        } catch (IOException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
     }
 }

src/main/java/io/ipfs/api/cbor/CborObject.java

@@ -305,6 +305,9 @@ public CborString(String value) {
 
         @Override
         public int compareTo(CborString cborString) {
+            int lenDiff = value.length() - cborString.value.length();
+            if (lenDiff != 0)
+                return lenDiff;
             return value.compareTo(cborString.value);
         }
 

src/test/java/io/ipfs/api/APITest.java

@@ -309,6 +309,20 @@ public void blockTest() {
         }
     }
 
+    @org.junit.Test
+    public void bulkBlockTest() {
+        try {
+            CborObject cbor = new CborObject.CborString("G'day IPFS!");
+            byte[] raw = cbor.toByteArray();
+            List<MerkleNode> bulkPut = ipfs.block.put(Arrays.asList(raw, raw, raw, raw, raw), Optional.of("cbor"));
+            List<Multihash> hashes = bulkPut.stream().map(m -> m.hash).collect(Collectors.toList());
+            byte[] result = ipfs.block.get(hashes.get(0));
+            System.out.println();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private static String toEscapedHex(byte[] in) throws IOException {
         StringBuilder res = new StringBuilder();
         for (byte b : in) {
@@ -325,20 +339,27 @@ private static String toEscapedHex(byte[] in) throws IOException {
     public void merkleLinkInMap() {
         try {
             Random r = new Random();
-            CborObject.CborByteArray target = new CborObject.CborByteArray(("g'day IPFS!" + r.nextInt()).getBytes());
+            CborObject.CborByteArray target = new CborObject.CborByteArray(("g'day IPFS!").getBytes());
             byte[] rawTarget = target.toByteArray();
             MerkleNode targetRes = ipfs.block.put(Arrays.asList(rawTarget), Optional.of("cbor")).get(0);
 
             CborObject.CborMerkleLink link = new CborObject.CborMerkleLink(targetRes.hash);
-            CborObject.CborMap source = CborObject.CborMap.build(Stream.of(link)
-                    .collect(Collectors.toMap(l -> l.target.toString(), l -> l)));
+            Map<String, CborObject> m = new TreeMap<>();
+            m.put("alink", link);
+            m.put("arr", new CborObject.CborList(Collections.emptyList()));
+            CborObject.CborMap source = CborObject.CborMap.build(m);
             byte[] rawSource = source.toByteArray();
             MerkleNode sourceRes = ipfs.block.put(Arrays.asList(rawSource), Optional.of("cbor")).get(0);
 
+            CborObject.fromByteArray(rawSource);
+
             List<Multihash> add = ipfs.pin.add(sourceRes.hash);
             ipfs.repo.gc();
             ipfs.repo.gc();
 
+            List<Multihash> refs = ipfs.refs(sourceRes.hash, true);
+            Assert.assertTrue("refs returns links", refs.contains(targetRes.hash));
+
             byte[] bytes = ipfs.block.get(targetRes.hash);
             Assert.assertTrue("same contents after GC", Arrays.equals(bytes, rawTarget));
             // These commands can be used to reproduce this on the command line
@@ -350,6 +371,44 @@ public void merkleLinkInMap() {
         }
     }
 
+    @org.junit.Test
+    public void recursiveRefs() {
+        try {
+            CborObject.CborByteArray leaf1 = new CborObject.CborByteArray(("G'day IPFS!").getBytes());
+            byte[] rawLeaf1 = leaf1.toByteArray();
+            MerkleNode leaf1Res = ipfs.block.put(Arrays.asList(rawLeaf1), Optional.of("cbor")).get(0);
+
+            CborObject.CborMerkleLink link = new CborObject.CborMerkleLink(leaf1Res.hash);
+            Map<String, CborObject> m = new TreeMap<>();
+            m.put("link1", link);
+            CborObject.CborMap source = CborObject.CborMap.build(m);
+            MerkleNode sourceRes = ipfs.block.put(Arrays.asList(source.toByteArray()), Optional.of("cbor")).get(0);
+
+            CborObject.CborByteArray leaf2 = new CborObject.CborByteArray(("G'day again, IPFS!").getBytes());
+            byte[] rawLeaf2 = leaf2.toByteArray();
+            MerkleNode leaf2Res = ipfs.block.put(Arrays.asList(rawLeaf2), Optional.of("cbor")).get(0);
+
+            Map<String, CborObject> m2 = new TreeMap<>();
+            m2.put("link1", new CborObject.CborMerkleLink(sourceRes.hash));
+            m2.put("link2", new CborObject.CborMerkleLink(leaf2Res.hash));
+            CborObject.CborMap source2 = CborObject.CborMap.build(m2);
+            MerkleNode rootRes = ipfs.block.put(Arrays.asList(source2.toByteArray()), Optional.of("cbor")).get(0);
+
+            List<Multihash> refs = ipfs.refs(rootRes.hash, false);
+            boolean correct = refs.contains(sourceRes.hash) && refs.contains(leaf2Res.hash) && refs.size() == 2;
+            Assert.assertTrue("refs returns links", correct);
+
+            List<Multihash> refsRecurse = ipfs.refs(rootRes.hash, true);
+            boolean correctRecurse = refs.contains(sourceRes.hash)
+                    && refs.contains(leaf1Res.hash)
+                    && refs.contains(leaf2Res.hash)
+                    && refs.size() == 3;
+            Assert.assertTrue("refs returns links", correct);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      *  Test that merkle links as a root object are followed during recursive pins
      */
@@ -385,6 +444,30 @@ public void rootMerkleLink() {
         }
     }
 
+    /**
+     *  Test that merkle links as a root object are followed during recursive pins
+     */
+    @org.junit.Test
+    public void rootNull() {
+        try {
+            CborObject.CborNull cbor = new CborObject.CborNull();
+            byte[] obj = cbor.toByteArray();
+            MerkleNode block = ipfs.block.put(Arrays.asList(obj), Optional.of("cbor")).get(0);
+            byte[] retrievedObj = ipfs.block.get(block.hash);
+            Assert.assertTrue("get inverse of put", Arrays.equals(retrievedObj, obj));
+
+            List<Multihash> add = ipfs.pin.add(block.hash);
+            ipfs.repo.gc();
+            ipfs.repo.gc();
+
+            // These commands can be used to reproduce this on the command line
+            String reproCommand1 = "printf \"" + toEscapedHex(obj) + "\" | ipfs block put --format=cbor";
+            System.out.println();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      *  Test that merkle links in a cbor list are followed during recursive pins
      */