From 766fc438559fb3ccd4d3282f5364eb60e33f85be Mon Sep 17 00:00:00 2001 From: Niko Lehto Date: Tue, 17 Sep 2019 09:28:30 +0200 Subject: [PATCH] Add support for VMware cursor encoding Supports both classic cursor type and alpha cursor type. In classic mode the server can send 'inverted' pixels for the cursor, our code does not support this but handles these pixels as opaque black. Co-authored-by: Samuel Mannehed --- kasmweb/core/encodings.js | 2 + kasmweb/core/rfb.js | 120 ++++++++++++++++++++++++++++ kasmweb/tests/test.rfb.js | 164 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+) diff --git a/kasmweb/core/encodings.js b/kasmweb/core/encodings.js index e34c132..dab2b22 100644 --- a/kasmweb/core/encodings.js +++ b/kasmweb/core/encodings.js @@ -51,6 +51,8 @@ export const encodings = { pseudoEncodingVideoScalingLevel9: -1987, pseudoEncodingVideoOutTimeLevel1: -1986, pseudoEncodingVideoOutTimeLevel100: -1887, + + pseudoEncodingVMwareCursor: 0x574d5664, }; export function encodingName(num) { diff --git a/kasmweb/core/rfb.js b/kasmweb/core/rfb.js index 8c6d35f..2f04152 100644 --- a/kasmweb/core/rfb.js +++ b/kasmweb/core/rfb.js @@ -1407,6 +1407,7 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingPreferBandwidth); if (this._fb_depth == 24) { + encs.push(encodings.pseudoEncodingVMwareCursor); encs.push(encodings.pseudoEncodingCursor); } @@ -1684,6 +1685,9 @@ export default class RFB extends EventTargetMixin { this._FBU.rects = 1; // Will be decreased when we return return true; + case encodings.pseudoEncodingVMwareCursor: + return this._handleVMwareCursor(); + case encodings.pseudoEncodingCursor: return this._handleCursor(); @@ -1714,6 +1718,122 @@ export default class RFB extends EventTargetMixin { } } + _handleVMwareCursor() { + const hotx = this._FBU.x; // hotspot-x + const hoty = this._FBU.y; // hotspot-y + const w = this._FBU.width; + const h = this._FBU.height; + if (this._sock.rQwait("VMware cursor encoding", 1)) { + return false; + } + + const cursor_type = this._sock.rQshift8(); + + this._sock.rQshift8(); //Padding + + let rgba; + const bytesPerPixel = 4; + + //Classic cursor + if (cursor_type == 0) { + //Used to filter away unimportant bits. + //OR is used for correct conversion in js. + const PIXEL_MASK = 0xffffff00 | 0; + rgba = new Array(w * h * bytesPerPixel); + + if (this._sock.rQwait("VMware cursor classic encoding", + (w * h * bytesPerPixel) * 2, 2)) { + return false; + } + + let and_mask = new Array(w * h); + for (let pixel = 0; pixel < (w * h); pixel++) { + and_mask[pixel] = this._sock.rQshift32(); + } + + let xor_mask = new Array(w * h); + for (let pixel = 0; pixel < (w * h); pixel++) { + xor_mask[pixel] = this._sock.rQshift32(); + } + + for (let pixel = 0; pixel < (w * h); pixel++) { + if (and_mask[pixel] == 0) { + //Fully opaque pixel + let bgr = xor_mask[pixel]; + let r = bgr >> 8 & 0xff; + let g = bgr >> 16 & 0xff; + let b = bgr >> 24 & 0xff; + + rgba[(pixel * bytesPerPixel) ] = r; //r + rgba[(pixel * bytesPerPixel) + 1 ] = g; //g + rgba[(pixel * bytesPerPixel) + 2 ] = b; //b + rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; //a + + } else if ((and_mask[pixel] & PIXEL_MASK) == + PIXEL_MASK) { + //Only screen value matters, no mouse colouring + if (xor_mask[pixel] == 0) { + //Transparent pixel + rgba[(pixel * bytesPerPixel) ] = 0x00; + rgba[(pixel * bytesPerPixel) + 1 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 2 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 3 ] = 0x00; + + } else if ((xor_mask[pixel] & PIXEL_MASK) == + PIXEL_MASK) { + //Inverted pixel, not supported in browsers. + //Fully opaque instead. + rgba[(pixel * bytesPerPixel) ] = 0x00; + rgba[(pixel * bytesPerPixel) + 1 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 2 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; + + } else { + //Unhandled xor_mask + rgba[(pixel * bytesPerPixel) ] = 0x00; + rgba[(pixel * bytesPerPixel) + 1 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 2 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; + } + + } else { + //Unhandled and_mask + rgba[(pixel * bytesPerPixel) ] = 0x00; + rgba[(pixel * bytesPerPixel) + 1 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 2 ] = 0x00; + rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; + } + } + + //Alpha cursor. + } else if (cursor_type == 1) { + if (this._sock.rQwait("VMware cursor alpha encoding", + (w * h * 4), 2)) { + return false; + } + + rgba = new Array(w * h * bytesPerPixel); + + for (let pixel = 0; pixel < (w * h); pixel++) { + let data = this._sock.rQshift32(); + + rgba[(pixel * 4) ] = data >> 8 & 0xff; //r + rgba[(pixel * 4) + 1 ] = data >> 16 & 0xff; //g + rgba[(pixel * 4) + 2 ] = data >> 24 & 0xff; //b + rgba[(pixel * 4) + 3 ] = data & 0xff; //a + } + + } else { + Log.Warn("The given cursor type is not supported: " + + cursor_type + " given."); + return false; + } + + this._updateCursor(rgba, hotx, hoty, w, h); + + return true; + } + _handleCursor() { const hotx = this._FBU.x; // hotspot-x const hoty = this._FBU.y; // hotspot-y diff --git a/kasmweb/tests/test.rfb.js b/kasmweb/tests/test.rfb.js index e8541b3..1c028cd 100644 --- a/kasmweb/tests/test.rfb.js +++ b/kasmweb/tests/test.rfb.js @@ -2132,6 +2132,170 @@ describe('Remote Frame Buffer Protocol Client', function () { }); }); + describe('the VMware Cursor pseudo-encoding handler', function () { + beforeEach(function () { + sinon.spy(client._cursor, 'change'); + }); + afterEach(function () { + client._cursor.change.resetHistory(); + }); + + it('should handle the VMware cursor pseudo-encoding', function () { + let data = [0x00, 0x00, 0xff, 0, + 0x00, 0xff, 0x00, 0, + 0x00, 0xff, 0x00, 0, + 0x00, 0x00, 0xff, 0]; + let rect = []; + push8(rect, 0); + push8(rect, 0); + + //AND-mask + for (let i = 0; i < data.length; i++) { + push8(rect, data[i]); + } + //XOR-mask + for (let i = 0; i < data.length; i++) { + push8(rect, data[i]); + } + + send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2, + encoding: 0x574d5664}], + [rect], client); + expect(client._FBU.rects).to.equal(0); + }); + + it('should handle insufficient cursor pixel data', function () { + + // Specified 14x23 pixels for the cursor, + // but only send 2x2 pixels worth of data + let w = 14; + let h = 23; + let data = [0x00, 0x00, 0xff, 0, + 0x00, 0xff, 0x00, 0]; + let rect = []; + + push8(rect, 0); + push8(rect, 0); + + //AND-mask + for (let i = 0; i < data.length; i++) { + push8(rect, data[i]); + } + //XOR-mask + for (let i = 0; i < data.length; i++) { + push8(rect, data[i]); + } + + send_fbu_msg([{ x: 0, y: 0, width: w, height: h, + encoding: 0x574d5664}], + [rect], client); + + // expect one FBU to remain unhandled + expect(client._FBU.rects).to.equal(1); + }); + + it('should update the cursor when type is classic', function () { + let and_mask = + [0xff, 0xff, 0xff, 0xff, //Transparent + 0xff, 0xff, 0xff, 0xff, //Transparent + 0x00, 0x00, 0x00, 0x00, //Opaque + 0xff, 0xff, 0xff, 0xff]; //Inverted + + let xor_mask = + [0x00, 0x00, 0x00, 0x00, //Transparent + 0x00, 0x00, 0x00, 0x00, //Transparent + 0x11, 0x22, 0x33, 0x44, //Opaque + 0xff, 0xff, 0xff, 0x44]; //Inverted + + let rect = []; + push8(rect, 0); //cursor_type + push8(rect, 0); //padding + let hotx = 0; + let hoty = 0; + let w = 2; + let h = 2; + + //AND-mask + for (let i = 0; i < and_mask.length; i++) { + push8(rect, and_mask[i]); + } + //XOR-mask + for (let i = 0; i < xor_mask.length; i++) { + push8(rect, xor_mask[i]); + } + + let expected_rgba = [0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x33, 0x22, 0x11, 0xff, + 0x00, 0x00, 0x00, 0xff]; + + send_fbu_msg([{ x: hotx, y: hoty, + width: w, height: h, + encoding: 0x574d5664}], + [rect], client); + + expect(client._cursor.change) + .to.have.been.calledOnce; + expect(client._cursor.change) + .to.have.been.calledWith(expected_rgba, + hotx, hoty, + w, h); + }); + + it('should update the cursor when type is alpha', function () { + let data = [0xee, 0x55, 0xff, 0x00, // bgra + 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0x22, + 0x00, 0xff, 0x00, 0x22, + 0x00, 0xff, 0x00, 0x22, + 0x00, 0x00, 0xff, 0xee]; + let rect = []; + push8(rect, 1); //cursor_type + push8(rect, 0); //padding + let hotx = 0; + let hoty = 0; + let w = 3; + let h = 2; + + for (let i = 0; i < data.length; i++) { + push8(rect, data[i]); + } + + let expected_rgba = [0xff, 0x55, 0xee, 0x00, + 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0x22, + 0x00, 0xff, 0x00, 0x22, + 0x00, 0xff, 0x00, 0x22, + 0xff, 0x00, 0x00, 0xee]; + + send_fbu_msg([{ x: hotx, y: hoty, + width: w, height: h, + encoding: 0x574d5664}], + [rect], client); + + expect(client._cursor.change) + .to.have.been.calledOnce; + expect(client._cursor.change) + .to.have.been.calledWith(expected_rgba, + hotx, hoty, + w, h); + }); + + it('should not update cursor when incorrect cursor type given', function () { + let rect = []; + push8(rect, 3); // invalid cursor type + push8(rect, 0); // padding + + client._cursor.change.resetHistory(); + send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2, + encoding: 0x574d5664}], + [rect], client); + + expect(client._cursor.change) + .to.not.have.been.called; + }); + }); + it('should handle the last_rect pseudo-encoding', function () { send_fbu_msg([{ x: 0, y: 0, width: 0, height: 0, encoding: -224}], [[]], client, 100); expect(client._FBU.rects).to.equal(0);