Use kubo library for file upload

package.json

@@ -51,10 +51,12 @@
     "image-size": "^0.8.1",
     "inquirer": "^6.2.1",
     "js-yaml": "^4.1.0",
+    "kubo-rpc-client": "^5.2.0",
     "listr": "^0.14.3",
     "lodash-es": "^4.17.21",
     "mime-types": "^2.1.24",
     "moment": "^2.27.0",
+    "multiformats": "^13.3.7",
     "prettier": "^2.1.2",
     "request": "^2.88.2",
     "rimraf": "^3.0.2",
@@ -93,5 +95,6 @@
   },
   "engines": {
     "node": ">=20.0.0"
-  }
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

src/commands/from_github.ts

@@ -12,12 +12,12 @@ import {
   GithubRelease
 } from "../utils/githubGetReleases.js";
 import { ipfsAddDirFromUrls } from "../releaseUploader/ipfsNode/addDirFromUrls.js";
-import { verifyIpfsConnection } from "../releaseUploader/ipfsNode/verifyConnection.js";
 import { CliGlobalOptions } from "../types.js";
 import { Manifest, defaultArch, releaseFiles } from "@dappnode/types";
 import { getLegacyImagePath } from "../utils/getLegacyImagePath.js";
 import { getImageFileName } from "../utils/getImageFileName.js";
 import { contentHashFileName, releaseFilesDefaultNames } from "../params.js";
+import { ReleaseUploaderIpfsNode } from "../releaseUploader/ipfsNode/index.js";
 
 interface CliCommandOptions extends CliGlobalOptions {
   repoSlug: string;
@@ -69,12 +69,11 @@ export async function fromGithubHandler({
   latest,
   version
 }: CliCommandOptions): Promise<{ releaseMultiHash: string }> {
-  // Parse options
-  const ipfsProvider = provider;
   // Assume incomplete repo slugs refer to DAppNode core packages
   if (!repoSlug.includes("/")) repoSlug = `dappnode/DNP_${repoSlug}`;
 
-  await verifyIpfsConnection(ipfsProvider);
+  const kuboClient = new ReleaseUploaderIpfsNode({ url: provider });
+  await kuboClient.testConnection();
 
   // Pick version interactively
   const release = await getSelectedGithubRelease({
@@ -135,7 +134,7 @@ export async function fromGithubHandler({
 
   const releaseMultiHash = await ipfsAddDirFromUrls(
     files,
-    ipfsProvider,
+    provider,
     onProgress
   );
 

src/releaseUploader/index.ts

@@ -13,7 +13,7 @@ export type ReleaseUploaderProvider =
   | {
       network: "ipfs";
       type: "node";
-      ipfsProvider: string;
+      url: string;
     }
   | {
       network: "ipfs";
@@ -88,7 +88,7 @@ export function cliArgsToReleaseUploaderProvider({
         return {
           network: "ipfs",
           type: "node",
-          ipfsProvider: contentProvider
+          url: contentProvider
         };
       }
 

src/releaseUploader/ipfsNode/addFromFs.ts

@@ -1,6 +1,6 @@
-import got from "got";
-import { normalizeIpfsProvider } from "./ipfsProvider.js";
-import { getFormDataFileUpload } from "../utils/formDataFileUpload.js";
+import { KuboRPCClient, ImportCandidate, AwaitIterable } from "kubo-rpc-client";
+import fs from "fs";
+import path from "path";
 
 /**
  * Uploads a directory or file from the fs
@@ -11,37 +11,85 @@ import { getFormDataFileUpload } from "../utils/formDataFileUpload.js";
  */
 export async function ipfsAddFromFs(
   dirOrFilePath: string,
-  ipfsProvider: string,
+  kuboClient: KuboRPCClient,
   onProgress?: (percent: number) => void
 ): Promise<string> {
-  // Create form and append all files recursively
-  const form = getFormDataFileUpload(dirOrFilePath);
+  // Calculate total size of all files before upload
+  function* getStatFiles(dir: string): Generator<string> {
+    const stat = fs.statSync(dir);
+    if (stat.isDirectory()) {
+      for (const entry of fs.readdirSync(dir)) {
+        yield* getStatFiles(path.join(dir, entry));
+      }
+    } else {
+      yield dir;
+    }
+  }
 
-  // Parse the ipfsProvider the a full base apiUrl
-  let lastPercent = -1;
-  const apiUrl = normalizeIpfsProvider(ipfsProvider);
-  const res = await got({
-    prefixUrl: apiUrl,
-    url: "api/v0/add",
-    method: "POST",
-    headers: form.getHeaders(),
-    body: form
-  }).on("uploadProgress", progress => {
-    // Report upload progress, and throttle to one update per percent point
-    // { percent: 0.9995998225975282, transferred: 733675762, total: 733969480 }
-    const currentRoundPercent = Math.round(100 * progress.percent);
-    if (lastPercent !== currentRoundPercent) {
-      lastPercent = currentRoundPercent;
-      if (onProgress) onProgress(progress.percent);
+  let totalSize = 0;
+  if (fs.statSync(dirOrFilePath).isDirectory()) {
+    for (const filePath of getStatFiles(dirOrFilePath)) {
+      const size = fs.statSync(filePath).size;
+      totalSize += size;
+      console.log(`[IPFS-UPLOAD] File: ${filePath}, Size: ${size}`);
+    }
+  } else {
+    totalSize = fs.statSync(dirOrFilePath).size;
+    console.log(
+      `[IPFS-UPLOAD] Single file: ${dirOrFilePath}, Size: ${totalSize}`
+    );
+  }
+  // Helper to recursively collect files from a directory
+  function* getFiles(
+    dir: string
+  ): Generator<{ path: string; content: fs.ReadStream }> {
+    const stat = fs.statSync(dir);
+    if (stat.isDirectory()) {
+      for (const entry of fs.readdirSync(dir)) {
+        yield* getFiles(path.join(dir, entry));
+      }
+    } else {
+      yield {
+        path: path.relative(path.dirname(dirOrFilePath), dir),
+        content: fs.createReadStream(dir)
+      };
     }
-  });
+  }
 
-  // res.body = '{"Name":"dir/file","Hash":"Qm...","Size":"2203"}\n{"Name":"dir","Hash":"Qm...","Size":"24622"}\n'
-  // Trim last \n, split entries by \n and then select the last which is the root directory
-  const lastFileUnparsed = res.body.trim().split("\n").slice(-1)[0];
-  if (!lastFileUnparsed) throw Error(`No files in response body ${res.body}`);
+  let files: AwaitIterable<ImportCandidate>;
+  if (fs.statSync(dirOrFilePath).isDirectory()) {
+    files = getFiles(dirOrFilePath);
+  } else {
+    files = (function* () {
+      yield {
+        path: path.basename(dirOrFilePath),
+        content: fs.createReadStream(dirOrFilePath)
+      };
+    })();
+  }
 
-  // Parse the JSON and return the hash of the root directory
-  const lastFile = JSON.parse(lastFileUnparsed);
-  return `/ipfs/${lastFile.Hash}`;
+  // Add files to IPFS
+  let lastCid = "";
+  let uploaded = 0;
+  console.log(`[IPFS-UPLOAD] Starting upload. Total size: ${totalSize}`);
+  for await (const result of kuboClient.addAll(files, {
+    wrapWithDirectory: true
+  })) {
+    lastCid = result.cid.toString();
+    uploaded += result.size || 0;
+    const percent = totalSize > 0 ? Math.min(uploaded / totalSize, 1) : 0;
+    console.log(
+      `[IPFS-UPLOAD] Uploaded chunk. CID: ${lastCid}, Path: ${result.path}, Size: ${result.size}`
+    );
+    console.log(
+      `[IPFS-UPLOAD] Progress: ${uploaded} bytes uploaded (${(
+        percent * 100
+      ).toFixed(2)}%)`
+    );
+    if (onProgress && totalSize > 0) {
+      onProgress(percent);
+    }
+  }
+  if (!lastCid) throw Error("No CID returned from IPFS add");
+  return `/ipfs/${lastCid}`;
 }

src/releaseUploader/ipfsNode/index.ts

@@ -1,13 +1,14 @@
 import { IReleaseUploader } from "../interface.js";
 import { ipfsAddFromFs } from "./addFromFs.js";
-import { verifyIpfsConnection } from "./verifyConnection.js";
+import { normalizeIpfsProvider } from "./ipfsProvider.js";
+import { create, KuboRPCClient } from "kubo-rpc-client";
 
 export class ReleaseUploaderIpfsNode implements IReleaseUploader {
   networkName = "IPFS node";
-  ipfsProvider: string;
+  kuboClient: KuboRPCClient;
 
-  constructor({ ipfsProvider }: { ipfsProvider: string }) {
-    this.ipfsProvider = ipfsProvider;
+  constructor({ url }: { url: string }) {
+    this.kuboClient = create({ url: normalizeIpfsProvider(url) });
   }
 
   async addFromFs({
@@ -17,10 +18,10 @@ export class ReleaseUploaderIpfsNode implements IReleaseUploader {
     dirPath: string;
     onProgress?: (percent: number) => void;
   }): Promise<string> {
-    return await ipfsAddFromFs(dirPath, this.ipfsProvider, onProgress);
+    return await ipfsAddFromFs(dirPath, this.kuboClient, onProgress);
   }
 
   async testConnection(): Promise<void> {
-    await verifyIpfsConnection(this.ipfsProvider);
+    await this.kuboClient.version();
   }
 }

src/releaseUploader/ipfsNode/ipfsProvider.ts

@@ -31,7 +31,12 @@ function parseIpfsProviderUrl(provider: string) {
 export function normalizeIpfsProvider(provider: string): string {
   const providerUrl = getIpfsProviderUrl(provider);
   const { host, port, protocol } = parseIpfsProviderUrl(providerUrl);
-  const fullUrl = `${protocol}://${host}:${port}`;
+  let fullUrl: string;
+  if (Number(port) === 443) {
+    fullUrl = `${protocol}://${host}`;
+  } else {
+    fullUrl = `${protocol}://${host}:${port}`;
+  }
   // #### TEMP: Make sure the URL is correct
   new URL(fullUrl);
   return fullUrl;