Efficiency improvements (#36)

* enable message compression
* convert all image to jpeg

Author: chemwolf6922

CHANGELOG.md

@@ -1,3 +1,16 @@
+# 0.2.0
+
+This is a breaking change and requires the backend with version 0.2.x.
+
+## What's new
+
+* **[Breaking]** Add adaptive application level message compression.
+* Convert all images to jpeg for better efficiency and compatibility.
+
+## Fixes
+
+## Changes
+
 # 0.1.7
 
 ## What's new

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tinywebui-webapp",
-  "version": "0.1.7",
+  "version": "0.2.0",
   "private": true,
   "type": "module",
   "scripts": {
@@ -10,6 +10,7 @@
     "test": "node --experimental-vm-modules node_modules/.bin/jest"
   },
   "dependencies": {
+    "@hpcc-js/wasm-zstd": "^1.11.0",
     "@radix-ui/react-select": "^2.2.5",
     "@radix-ui/react-slot": "^1.2.3",
     "class-variance-authority": "^0.7.1",

src/app/chat/user-input.tsx

@@ -7,6 +7,54 @@ import React from "react";
 import * as ServerTypes from "@/sdk/types/IServer";
 import { cn } from "@/lib/utils";
 
+const readBlobAsDataUrl = (blob: Blob) => new Promise<string>((resolve, reject) => {
+  const reader = new FileReader();
+  reader.onload = () => resolve(reader.result as string);
+  reader.onerror = () => reject(reader.error);
+  reader.readAsDataURL(blob);
+});
+
+const reencodeImageToJpeg = (file: File) => new Promise<string>((resolve, reject) => {
+  const objectUrl = URL.createObjectURL(file);
+  const img = new Image();
+  img.onload = () => {
+    try {
+      const canvas = document.createElement("canvas");
+      canvas.width = img.naturalWidth || img.width;
+      canvas.height = img.naturalHeight || img.height;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) {
+        reject(new Error("Canvas context unavailable"));
+        return;
+      }
+      ctx.drawImage(img, 0, 0);
+      canvas.toBlob((blob) => {
+        URL.revokeObjectURL(objectUrl);
+        if (!blob) {
+          reject(new Error("Failed to encode JPEG"));
+          return;
+        }
+        readBlobAsDataUrl(blob).then(resolve).catch(reject);
+      }, "image/jpeg", 0.92);
+    } catch (err) {
+      URL.revokeObjectURL(objectUrl);
+      reject(err);
+    }
+  };
+  img.onerror = () => {
+    URL.revokeObjectURL(objectUrl);
+    reject(new Error("Failed to load pasted image"));
+  };
+  img.src = objectUrl;
+});
+
+const ensureJpegDataUrl = (file: File) => {
+  if (file.type === "image/jpeg") {
+    return readBlobAsDataUrl(file);
+  }
+  return reencodeImageToJpeg(file).catch(() => readBlobAsDataUrl(file));
+};
+
 interface UserInputProps {
   onUserMessage: (message: ServerTypes.Message) => void;
   /** This controls the send button. Not the editor. */
@@ -167,12 +215,7 @@ export function UserInput({ onUserMessage, inputEnabled, initialMessage, editorH
       if (item.kind === "file" && item.type.startsWith("image/")) {
         const file = item.getAsFile();
         if (!file) continue;
-        filePromises.push(new Promise((resolve, reject) => {
-          const reader = new FileReader();
-          reader.onload = () => resolve(reader.result as string);
-          reader.onerror = () => reject(reader.error);
-          reader.readAsDataURL(file);
-        }));
+        filePromises.push(ensureJpegDataUrl(file));
       }
     }
     if (filePromises.length > 0) {

src/sdk/session/secure-session.ts

@@ -7,12 +7,20 @@ import { HandshakeMessage, HandshakeMessageType } from "../cipher/handshake-mess
 import * as IServer from "../types/IServer";
 import { isSecureContext } from "./secure-context";
 import { sodium } from "../cipher/sodium";
+import { Zstd } from "@hpcc-js/wasm-zstd";
+
+const zstd = await Zstd.load();
 
 enum ProtocolType {
     Password = 0,
     Psk = 1,
 };
 
+enum CompressionType {
+    None =0,
+    Zstd = 1,
+};
+
 export class Connection extends IConnection {
     #connection: IConnection;
     #username: string | undefined;
@@ -123,6 +131,7 @@ export class Connection extends IConnection {
     }
 
     send(data: Uint8Array): void {
+        data = this.#compressData(data);
         if (!this.#turnOffEncryption) {
             if (this.#encryptor === undefined) {
                 throw new Error("Encryption not established");
@@ -143,7 +152,41 @@ export class Connection extends IConnection {
             }
             data = this.#decryptor.decrypt(data);
         }
+        data = this.#decompressData(data);
         return data;
     }
+
+    #compressData(data: Uint8Array): Uint8Array {
+        if (data.length < 100) {
+            /** Not worth it */
+            return this.#formCompressedData(data, CompressionType.None);
+        }
+        const compressed = zstd.compress(data, 3);
+        if (compressed.length + 1 >= data.length) {
+            /** Compression not effective */
+            return this.#formCompressedData(data, CompressionType.None);
+        }
+        return this.#formCompressedData(compressed, CompressionType.Zstd);
+    }
+
+    #formCompressedData(data: Uint8Array, type: CompressionType): Uint8Array {
+        const result = new Uint8Array(1 + data.length);
+        result[0] = type;
+        result.set(data, 1);
+        return result;
+    }
+
+    #decompressData(data: Uint8Array): Uint8Array {
+        const type = data[0];
+        const compressedData = data.subarray(1);
+        switch (type) {
+            case CompressionType.None:
+                return compressedData;
+            case CompressionType.Zstd:
+                return zstd.decompress(compressedData);
+            default:
+                throw new Error("Unknown compression type");
+        }
+    }
 }