From 1a57d41a1b5dd411efa592fa647b3bc58e499cf1 Mon Sep 17 00:00:00 2001
From: Michael Jumper <mjumper@apache.org>
Date: Wed, 29 Jan 2025 09:58:01 -0800
Subject: [PATCH] GUACAMOLE-377: Use ImageDecoder to decode images while they
 are being received (if possible).

---
 .../main/webapp/modules/ArrayBufferReader.js  | 26 ++++++--
 .../src/main/webapp/modules/Display.js        | 44 ++++++++++---
 .../src/main/webapp/modules/InputStream.js    | 62 +++++++++++++++++++
 3 files changed, 116 insertions(+), 16 deletions(-)

diff --git a/guacamole-common-js/src/main/webapp/modules/ArrayBufferReader.js b/guacamole-common-js/src/main/webapp/modules/ArrayBufferReader.js
index 2d542a6dd..ea3a00952 100644
--- a/guacamole-common-js/src/main/webapp/modules/ArrayBufferReader.js
+++ b/guacamole-common-js/src/main/webapp/modules/ArrayBufferReader.js
@@ -39,13 +39,27 @@ Guacamole.ArrayBufferReader = function(stream) {
     // Receive blobs as array buffers
     stream.onblob = function(data) {
 
-        // Convert to ArrayBuffer
-        var binary = window.atob(data);
-        var arrayBuffer = new ArrayBuffer(binary.length);
-        var bufferView = new Uint8Array(arrayBuffer);
+        var arrayBuffer, bufferView;
 
-        for (var i=0; i<binary.length; i++)
-            bufferView[i] = binary.charCodeAt(i);
+        // Use native methods for directly decoding base64 to an array buffer
+        // when possible
+        if (Uint8Array.fromBase64) {
+            bufferView = Uint8Array.fromBase64(data);
+            arrayBuffer = bufferView.buffer;
+        }
+
+        // Rely on binary strings and manual conversions where native methods
+        // like fromBase64() are not available
+        else {
+
+            var binary = window.atob(data);
+            arrayBuffer = new ArrayBuffer(binary.length);
+            bufferView = new Uint8Array(arrayBuffer);
+
+            for (var i=0; i<binary.length; i++)
+                bufferView[i] = binary.charCodeAt(i);
+
+        }
 
         // Call handler, if present
         if (guac_reader.ondata)
diff --git a/guacamole-common-js/src/main/webapp/modules/Display.js b/guacamole-common-js/src/main/webapp/modules/Display.js
index 57e871940..dcb1e320c 100644
--- a/guacamole-common-js/src/main/webapp/modules/Display.js
+++ b/guacamole-common-js/src/main/webapp/modules/Display.js
@@ -480,7 +480,10 @@ Guacamole.Display = function() {
         this.unblock = function() {
             if (task.blocked) {
                 task.blocked = false;
-                __flush_frames();
+
+                if (frames.length)
+                    __flush_frames();
+
             }
         };
 
@@ -905,17 +908,38 @@ Guacamole.Display = function() {
      */
     this.drawStream = function drawStream(layer, x, y, stream, mimetype) {
 
-        // If createImageBitmap() is available, load the image as a blob so
-        // that function can be used
-        if (window.createImageBitmap) {
-            var reader = new Guacamole.BlobReader(stream, mimetype);
-            reader.onend = function drawImageBlob() {
-                guac_display.drawBlob(layer, x, y, reader.getBlob());
-            };
+        // Leverage ImageDecoder to decode the image stream as it is received
+        // whenever possible, as this reduces latency that might otherwise be
+        // caused by waiting for the full image to be received
+        if (window.ImageDecoder && window.ReadableStream) {
+
+            var imageDecoder = new ImageDecoder({
+                type: mimetype,
+                data: stream.toReadableStream()
+            });
+
+            var decodedFrame = null;
+
+            // Draw image once loaded
+            var task = scheduleTask(function drawImageBitmap() {
+                layer.drawImage(x, y, decodedFrame);
+            }, true);
+
+            imageDecoder.decode({ completeFramesOnly: true }).then(function bitmapLoaded(result) {
+                decodedFrame = result.image;
+                task.unblock();
+            });
+
         }
 
-        // Lacking createImageBitmap(), fall back to data URIs and the Image
-        // object
+        // NOTE: We do not use Blobs and createImageBitmap() here, as doing so
+        // is very latent compared to the old data URI method and the new
+        // ImageDecoder object. The new ImageDecoder object is currently
+        // supported by most browsers, with other browsers being much faster if
+        // data URIs are used. The iOS version of Safari is particularly laggy
+        // if Blobs and createImageBitmap() are used instead.
+
+        // Lacking ImageDecoder, fall back to data URIs and the Image object
         else {
             var reader = new Guacamole.DataURIReader(stream, mimetype);
             reader.onend = function drawImageDataURI() {
diff --git a/guacamole-common-js/src/main/webapp/modules/InputStream.js b/guacamole-common-js/src/main/webapp/modules/InputStream.js
index 7bda1330b..58798a700 100644
--- a/guacamole-common-js/src/main/webapp/modules/InputStream.js
+++ b/guacamole-common-js/src/main/webapp/modules/InputStream.js
@@ -76,4 +76,66 @@ Guacamole.InputStream = function(client, index) {
         client.sendAck(guac_stream.index, message, code);
     };
 
+    /**
+     * Creates a new ReadableStream that receives the data sent to this stream
+     * by the Guacamole server. This function may be invoked at most once per
+     * stream, and invoking this function will overwrite any installed event
+     * handlers on this stream.
+     * 
+     * A ReadableStream is a JavaScript object defined by the "Streams"
+     * standard. It is supported by most browsers, but not necessarily all
+     * browsers. The caller should verify this support is present before
+     * invoking this function. The behavior of this function when the browser
+     * does not support ReadableStream is not defined.
+     *
+     * @see {@link https://streams.spec.whatwg.org/#rs-class}
+     *
+     * @returns {!ReadableStream}
+     *     A new ReadableStream that receives the bytes sent along this stream
+     *     by the Guacamole server.
+     */
+    this.toReadableStream = function toReadableStream() {
+        return new ReadableStream({
+            type: 'bytes',
+            start: function startStream(controller) {
+
+                var reader = new Guacamole.ArrayBufferReader(guac_stream);
+
+                // Provide any received blocks of data to the ReadableStream
+                // controller, such that they will be read by whatever is
+                // consuming the ReadableStream
+                reader.ondata = function dataReceived(data) {
+
+                    if (controller.byobRequest) {
+
+                        var view = controller.byobRequest.view;
+                        var length = Math.min(view.byteLength, data.byteLength);
+                        var byobBlock = new Uint8Array(data, 0, length);
+
+                        view.buffer.set(byobBlock);
+                        controller.byobRequest.respond(length);
+
+                        if (length < data.byteLength) {
+                            controller.enqueue(data.slice(length));
+                        }
+
+                    }
+
+                    else {
+                        controller.enqueue(new Uint8Array(data));
+                    }
+
+                };
+
+                // Notify the ReadableStream when the end of the stream is
+                // reached
+                reader.onend = function dataComplete() {
+                    controller.close();
+                };
+
+            }
+        });
+
+    };
+
 };