diff --git a/guacamole-common-js/src/main/webapp/modules/KeyEventInterpreter.js b/guacamole-common-js/src/main/webapp/modules/KeyEventInterpreter.js index 957a90c6b..3e37629c1 100644 --- a/guacamole-common-js/src/main/webapp/modules/KeyEventInterpreter.js +++ b/guacamole-common-js/src/main/webapp/modules/KeyEventInterpreter.js @@ -22,18 +22,24 @@ var Guacamole = Guacamole || {}; /** * An object that will accept raw key events and produce human readable text * batches, seperated by at least `batchSeperation` milliseconds, which can be - * retrieved through the onBatch callback or by calling getCurrentBatch(). + * retrieved through the onbatch callback or by calling getCurrentBatch(). * * NOTE: The event processing logic and output format is based on the `guaclog` * tool, with the addition of batching support. * * @constructor + * * @param {number} [batchSeperation=5000] * The minimum number of milliseconds that must elapse between subsequent * batches of key-event-generated text. If 0 or negative, no splitting will * occur, resulting in a single batch for all provided key events. + * + * @param {number} [startTimestamp=0] + * The starting timestamp for the recording being intepreted. If provided, + * the timestamp of each intepreted event will be relative to this timestamp. + * If not provided, the raw recording timestamp will be used. */ -Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { +Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation, startTimestamp) { /** * Reference to this Guacamole.KeyEventInterpreter. @@ -47,6 +53,10 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { if (batchSeperation === undefined || batchSeperation === null) batchSeperation = 5000; + // Default to 0 seconds to keep the raw timestamps + if (startTimestamp === undefined || startTimestamp === null) + startTimestamp = 0; + /** * A definition for a known key. * @@ -212,21 +222,14 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { var pressedKeys = {}; /** - * A human-readable representation of all keys pressed since the last keyframe. + * The current key event batch, containing a representation of all key + * events processed since the end of the last batch passed to onbatch. + * Null if no key events have been processed yet. * * @private - * @type {String} + * @type {!KeyEventBatch} */ - var currentTypedValue = ''; - - /** - * The timestamp of the key event that started the most recent batch of - * text content. If 0, no key events have been processed yet. - * - * @private - * @type {Number} - */ - var lastTextTimestamp = 0; + var currentBatch = null; /** * The timestamp of the most recent key event processed. @@ -265,9 +268,9 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { * @param {Number} keysym * The keysym to produce a UTF-8 KeyDefinition for, if valid. * - * @returns - * Return a KeyDefinition for the provided keysym, if it it's a valid - * UTF-8 keysym, or null otherwise. + * @returns {KeyDefinition} + * A KeyDefinition for the provided keysym, if it's a valid UTF-8 + * keysym, or null otherwise. */ function getUnicodeKeyDefinition(keysym) { @@ -279,7 +282,7 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { var mask; var bytes; - /* Determine size and initial byte mask */ + // Determine size and initial byte mask if (codepoint <= 0x007F) { mask = 0x00; bytes = 1; @@ -309,7 +312,7 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { var name = new TextDecoder("utf-8").decode(byteArray); // Create and return the definition - return new KeyDefinition({keysym: keysym.toString(), name: name, value: name, modifier: false}); + return new KeyDefinition({keysym: keysym, name: name, value: name, modifier: false}); } @@ -320,7 +323,7 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { * @param {Number} keysym * The keysym to return a KeyDefinition for. * - * @returns + * @returns {KeyDefinition} * A KeyDefinition corresponding to the provided keysym. */ function getKeyDefinitionByKeysym(keysym) { @@ -350,24 +353,19 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { * previous key event. * * @event - * @param {!String} text - * The typed text associated with the batch of text. - * - * @param {!number} timestamp - * The raw recording timestamp associated with the first key event - * that started this batch of text. + * @param {!Guacamole.KeyEventInterpreter.KeyEventBatch} */ - interpreter.onBatch = null; + this.onbatch = null; /** * Handles a raw key event, potentially appending typed text to the - * current batch, and calling onBatch with the current batch, if the + * current batch, and calling onbatch with the current batch, if the * callback is set and a new batch is about to be started. * * @param {!string[]} args * The arguments of the key event. */ - interpreter.handleKeyEvent = function handleKeyEvent(args) { + this.handleKeyEvent = function handleKeyEvent(args) { // The X11 keysym var keysym = parseInt(args[0]); @@ -379,10 +377,8 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { var timestamp = parseInt(args[2]); // If no current batch exists, start a new one now - if (!lastTextTimestamp) { - lastTextTimestamp = timestamp; - lastKeyEvent = timestamp; - } + if (!currentBatch) + currentBatch = new Guacamole.KeyEventInterpreter.KeyEventBatch(); // Only switch to a new batch of text if sufficient time has passed // since the last key event @@ -394,12 +390,11 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { // Call the handler with the current batch of text and the timestamp // at which the current batch started - if (currentTypedValue && interpreter.onBatch) - interpreter.onBatch(currentTypedValue, lastTextTimestamp); + if (currentBatch.events.length && interpreter.onbatch) + interpreter.onbatch(currentBatch); // Move on to the next batch of text - currentTypedValue = ''; - lastTextTimestamp = 0; + currentBatch = new Guacamole.KeyEventInterpreter.KeyEventBatch(); } @@ -417,9 +412,11 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { // (non-modifier) key is pressed else if (pressed) { + var relativeTimestap = timestamp - startTimestamp; + if (isShortcut()) { - currentTypedValue += '<'; + var shortcutText = '<'; var firstKey = true; @@ -431,30 +428,56 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { // Print name of key if (firstKey) { - currentTypedValue += pressedKeyDefinition.name; + shortcutText += pressedKeyDefinition.name; firstKey = false; } else - currentTypedValue += ('+' + pressedKeyDefinition.name); + shortcutText += ('+' + pressedKeyDefinition.name); } // Finally, append the printable key to close the shortcut - currentTypedValue += ('+' + keyDefinition.name + '>') + shortcutText += ('+' + keyDefinition.name + '>') + + // Add the shortcut to the current batch + currentBatch.simpleValue += shortcutText; + currentBatch.events.push(new Guacamole.KeyEventInterpreter.KeyEvent( + shortcutText, false, relativeTimestap)); } // Print the key itself else { + var keyText; + var typed; + // Print the value if explicitly defined - if (keyDefinition.value != null) - currentTypedValue += keyDefinition.value; + if (keyDefinition.value != null) { + + keyText = keyDefinition.value; + typed = true; + + } // Otherwise print the name - else - currentTypedValue += ('<' + keyDefinition.name + '>'); + else { + + keyText = ('<' + keyDefinition.name + '>'); + + // While this is a representation for a single character, + // the key text is the name of the key, not the actual + // character itself + typed = false; + + } + + // Add the key to the current batch + currentBatch.simpleValue += keyText; + currentBatch.events.push(new Guacamole.KeyEventInterpreter.KeyEvent( + keyText, typed, relativeTimestap)); + } } @@ -466,22 +489,90 @@ Guacamole.KeyEventInterpreter = function KeyEventInterpreter(batchSeperation) { * incomplete, as more key events might be processed before the next * batch starts. * - * @returns + * @returns {Guacamole.KeyEventInterpreter.KeyEventBatch} * The current batch of text. */ - interpreter.getCurrentText = function getCurrentText() { - return currentTypedValue; + this.getCurrentBatch = function getCurrentBatch() { + return currentBatch; } +} + +/** + * A granular description of an extracted key event, including a human-readable + * text representation of the event, whether the event is directly typed or not, + * and the timestamp when the event occured. + * + * @constructor + * @param {!String} text + * A human-readable representation of the event. + * + * @param {!boolean} typed + * True if this event represents a directly-typed character, or false + * otherwise. + * + * @param {!Number} timestamp + * The timestamp from the recording when this event occured. + */ +Guacamole.KeyEventInterpreter.KeyEvent = function KeyEvent(text, typed, timestamp) { /** - * Return the recording timestamp associated with the start of the - * current batch of typed text. + * A human-readable representation of the event. If a printable character + * was directly typed, this will just be that character. Otherwise it will + * be a string describing the event. * - * @returns - * The recording timestamp at which the current batch started. + * @type {!String} */ - interpreter.getCurrentTimestamp = function getCurrentTimestamp() { - return lastTextTimestamp; - } + this.text = text; + /** + * True if this text of this event is exactly a typed character, or false + * otherwise. + * + * @type {!boolean} + */ + this.typed = typed; + + /** + * The timestamp from the recording when this event occured. If a + * `startTimestamp` value was provided to the interpreter constructor, this + * will be relative to start of the recording. If not, it will be the raw + * timestamp from the key event. + * + * @type {!Number} + */ + this.timestamp = timestamp; + +}; + +/** + * A series of intepreted key events, seperated by at least the configured + * batchSeperation value from any other key events in the recording corresponding + * to the interpreted key events. A batch will always consist of at least one key + * event, and an associated simplified representation of the event(s). + * + * @constructor + * @param {!Guacamole.KeyEventInterpreter.KeyEvent[]} events + * The interpreted key events for this batch. + * + * @param {!String} simpleValue + * The simplified, human-readable value representing the key events for + * this batch. + */ +Guacamole.KeyEventInterpreter.KeyEventBatch = function KeyEventBatch(events, simpleValue) { + + /** + * All key events for this batch. + * + * @type {!Guacamole.KeyEventInterpreter.KeyEvent[]} + */ + this.events = events || []; + + /** + * The simplified, human-readable value representing the key events for + * this batch, equivalent to concatenating the `text` field of all key + * events in the batch. + * + * @type {!String} + */ + this.simpleValue = simpleValue || ''; } diff --git a/guacamole-common-js/src/main/webapp/modules/SessionRecording.js b/guacamole-common-js/src/main/webapp/modules/SessionRecording.js index 18db916d4..619f78fba 100644 --- a/guacamole-common-js/src/main/webapp/modules/SessionRecording.js +++ b/guacamole-common-js/src/main/webapp/modules/SessionRecording.js @@ -104,16 +104,6 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) */ var KEYFRAME_TIME_INTERVAL = 5000; - /** - * The minimum number of milliseconds which must elapse between key events - * before text can be split across multiple frames. - * - * @private - * @constant - * @type {Number} - */ - var TYPED_TEXT_INTERVAL = 5000; - /** * All frames parsed from the provided blob. * @@ -387,23 +377,35 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) /** * A key event interpreter to split all key events in this recording into - * human-readable batches of text. + * human-readable batches of text. Constrcution is deferred until the first + * event is processed, to enable recording-relative timestamps. * * @type {!Guacamole.KeyEventInterpreter} */ - var keyEventInterpreter = new Guacamole.KeyEventInterpreter(); + var keyEventInterpreter = null; - // Pass through any received batches to the recording ontext handler - keyEventInterpreter.onBatch = function onBatch(text, timestamp) { + /** + * Initialize the key interpreter. This function should be called only once + * with the first timestamp in the recording as an argument. + * + * @private + * @param {!number} startTimestamp + * The timestamp of the first frame in the recording, i.e. the start of + * the recording. + */ + function initializeKeyInterpreter(startTimestamp) { - // Don't call the callback if it was never set - if (!recording.ontext) - return; + keyEventInterpreter = new Guacamole.KeyEventInterpreter(null, startTimestamp); - // Convert to a recording-relative timestamp and pass through - recording.ontext(text, toRelativeTimestamp(timestamp)); + // Pass through any received batches to the recording ontext handler + keyEventInterpreter.onbatch = function onbatch(batch) { - }; + // Pass the batch through if a handler is set + if (recording.ontext) + recording.ontext(batch); + + }; + } /** * Handles a newly-received instruction, whether from the main Blob or a @@ -436,6 +438,11 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) frames.push(frame); frameStart = frameEnd; + // If this is the first frame, intialize the key event interpreter + // with the timestamp of the first frame + if (frames.length === 1) + initializeKeyInterpreter(timestamp); + // This frame should eventually become a keyframe if enough data // has been processed and enough recording time has elapsed, or if // this is the absolute first frame @@ -443,6 +450,7 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) && timestamp - frames[lastKeyframe].timestamp >= KEYFRAME_TIME_INTERVAL)) { frame.keyframe = true; lastKeyframe = frames.length - 1; + } // Notify that additional content is available @@ -521,10 +529,9 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) // If there's any typed text that's yet to be sent to the ontext // handler, send it now - var text = keyEventInterpreter.getCurrentText(); - var timestamp = keyEventInterpreter.getCurrentTimestamp(); - if (text && recording.ontext) - recording.ontext(text, toRelativeTimestamp(timestamp)); + var batch = keyEventInterpreter.getCurrentBatch(); + if (batch && recording.ontext) + recording.ontext(batch); // Consider recording loaded if tunnel has closed without errors if (!errorEncountered) @@ -916,11 +923,8 @@ Guacamole.SessionRecording = function SessionRecording(source, refreshInterval) * is available. * * @event - * @param {!String} text - * The typed text associated with the batch of text. - * - * @param {!number} timestamp - * The relative timestamp associated with the batch of text. + * @param {!Guacamole.KeyEventInterpreter.KeyEventBatch} batch + * The batch of extracted text. */ this.ontext = null; diff --git a/guacamole/src/main/frontend/src/app/player/directives/player.js b/guacamole/src/main/frontend/src/app/player/directives/player.js index 43b1c8ec2..7e50dd7af 100644 --- a/guacamole/src/main/frontend/src/app/player/directives/player.js +++ b/guacamole/src/main/frontend/src/app/player/directives/player.js @@ -77,9 +77,6 @@ */ angular.module('player').directive('guacPlayer', ['$injector', function guacPlayer($injector) { - // Required types - const TextBatch = $injector.get('TextBatch'); - // Required services const playerTimeService = $injector.get('playerTimeService'); @@ -151,7 +148,7 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay /** * Any batches of text typed during the recording. * - * @type {TextBatch[]} + * @type {Guacamole.KeyEventInterpeter.KeyEventBatch[]} */ $scope.textBatches = []; @@ -277,6 +274,7 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay resumeAfterSeekRequest && $scope.recording.play(); $scope.recording.seek($scope.playbackPosition, function seekComplete() { + $scope.seekPosition = null; $scope.operationMessage = null; $scope.$evalAsync(); }); @@ -357,8 +355,8 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay }; // Append any extracted batches of typed text - $scope.recording.ontext = function appendTextBatch(text, timestamp) { - $scope.textBatches.push({text, timestamp}); + $scope.recording.ontext = function appendTextBatch(batch) { + $scope.textBatches.push(batch); } // Notify listeners when current position within the recording diff --git a/guacamole/src/main/frontend/src/app/player/directives/textView.js b/guacamole/src/main/frontend/src/app/player/directives/textView.js index 2a9fbb303..3ff5b1b17 100644 --- a/guacamole/src/main/frontend/src/app/player/directives/textView.js +++ b/guacamole/src/main/frontend/src/app/player/directives/textView.js @@ -25,9 +25,6 @@ const fuzzysort = require('fuzzysort') angular.module('player').directive('guacPlayerTextView', ['$injector', function guacPlayer($injector) { - // Required types - const TextBatch = $injector.get('TextBatch'); - // Required services const playerTimeService = $injector.get('playerTimeService'); @@ -41,7 +38,7 @@ angular.module('player').directive('guacPlayerTextView', /** * All the batches of text extracted from this recording. * - * @type {!TextBatch[]} + * @type {!Guacamole.KeyEventInterpeter.KeyEventBatch[]} */ textBatches : '=', @@ -77,7 +74,7 @@ angular.module('player').directive('guacPlayerTextView', * The text batches that match the current search phrase, or all * batches if no search phrase is set. * - * @type {!TextBatch[]} + * @type {!Guacamole.KeyEventInterpeter.KeyEventBatch[]} */ $scope.filteredBatches = $scope.textBatches; @@ -111,7 +108,7 @@ angular.module('player').directive('guacPlayerTextView', // batches for it if (searchPhrase) $scope.filteredBatches = fuzzysort.go( - searchPhrase, $scope.textBatches, {key: 'text'}) + searchPhrase, $scope.textBatches, {key: 'simpleValue'}) .map(result => result.obj); // Otherwise, do not filter the batches diff --git a/guacamole/src/main/frontend/src/app/player/services/playerTimeService.js b/guacamole/src/main/frontend/src/app/player/services/playerTimeService.js index b005cbc20..919c4eec9 100644 --- a/guacamole/src/main/frontend/src/app/player/services/playerTimeService.js +++ b/guacamole/src/main/frontend/src/app/player/services/playerTimeService.js @@ -17,6 +17,33 @@ * under the License. */ + /* + * NOTE: This session recording player implementation is based on the Session + * Recording Player for Glyptodon Enterprise which is available at + * https://github.com/glyptodon/glyptodon-enterprise-player under the + * following license: + * + * Copyright (C) 2019 Glyptodon, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + /** * A service for formatting time, specifically for the recording player. */ diff --git a/guacamole/src/main/frontend/src/app/player/styles/textView.css b/guacamole/src/main/frontend/src/app/player/styles/textView.css index 3a73c940e..d7502b188 100644 --- a/guacamole/src/main/frontend/src/app/player/styles/textView.css +++ b/guacamole/src/main/frontend/src/app/player/styles/textView.css @@ -17,33 +17,6 @@ * under the License. */ -/* - * NOTE: This session recording player implementation is based on the Session - * Recording Player for Glyptodon Enterprise which is available at - * https://github.com/glyptodon/glyptodon-enterprise-player under the - * following license: - * - * Copyright (C) 2019 Glyptodon, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - .text-batches { display: flex; @@ -113,3 +86,25 @@ margin: 0.25em; } + +.text-batches .text { + + display: block; + margin: 0; + padding: 0; + max-width: 100%; + overflow-wrap: break-word; + +} + +.text-batches .text .not-typed { + + font-weight: bold; + +} + +.text-batches .text .future { + + color: dimgray; + +} diff --git a/guacamole/src/main/frontend/src/app/player/templates/textView.html b/guacamole/src/main/frontend/src/app/player/templates/textView.html index 1d30da58f..4cef97639 100644 --- a/guacamole/src/main/frontend/src/app/player/templates/textView.html +++ b/guacamole/src/main/frontend/src/app/player/templates/textView.html @@ -15,9 +15,15 @@ translate-values="{RESULTS: filteredBatches.length}">
-
-
{{ formatTime(batch.timestamp) }}
-
{{ batch.text }}
+
+
{{ formatTime(batch.events[0].timestamp) }}
+
+ {{ event.text }} +
diff --git a/guacamole/src/main/frontend/src/app/player/types/TextBatch.js b/guacamole/src/main/frontend/src/app/player/types/TextBatch.js deleted file mode 100644 index fbe92c7d2..000000000 --- a/guacamole/src/main/frontend/src/app/player/types/TextBatch.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Service which defines the TextBatch class. - */ -angular.module('player').factory('TextBatch', [function defineTextBatch() { - - /** - * A batch of text associated with a recording. The batch consists of a - * string representation of the text that would be typed based on the key - * events in the recording, as well as a timestamp when the batch started. - * - * @constructor - * @param {TextBatch|Object} [template={}] - * The object whose properties should be copied within the new TextBatch. - */ - const TextBatch = function TextBatch(template) { - - // Use empty object by default - template = template || {}; - - /** - * The text that was typed in this batch. - * - * @type String - */ - this.text = template.text; - - /** - * The timestamp at which the batch of text was typed. - * - * @type Number - */ - this.timestamp = template.timestamp; - - }; - - return TextBatch; - -}]);