publicWagsHome/KasmVNC
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restore x264 as a fallback

Browse Source
This commit is contained in:
Lauri Kasanen
2021-11-22 11:41:22 +02:00
parent b09e1b907f
commit ac56f5566b
3 changed files with 350 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

538
common/rfb/TightX264Encoder.cxx
View File

@@ -28,6 +28,7 @@
#include <rfb/TightConstants.h>
#include <webp/encode.h>
#include <x264.h>
#include "nvidia.h"
#include "mp4.h"
@@ -42,11 +43,23 @@ static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
TightX264Encoder::TightX264Encoder(SConnection* conn, EncCache *cache_, uint8_t cacheType_) :
Encoder(conn, encodingTight, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1),
keyframe(true), mux(NULL), muxstate(NULL), framectr(0),
nvidia_init_done(false),
keyframe(true), enc(NULL), params(NULL), mux(NULL), muxstate(NULL), framectr(0),
nvidia_init_done(false), using_nvidia(true),
encCache(cache_), cacheType(cacheType_),
framebuf(NULL), framelen(0), bitbuf(NULL), myw(0), myh(0)
{
params = new x264_param_t;
x264_param_default_preset(params, "veryfast", "zerolatency");
params->i_threads = X264_THREADS_AUTO;
params->i_fps_num = params->i_keyint_max = rfb::Server::frameRate;
params->i_fps_den = 1;
params->rc.i_rc_method = X264_RC_ABR;
params->rc.i_bitrate = rfb::Server::x264Bitrate;
params->i_csp = X264_CSP_I420;
params->i_log_level = X264_LOG_WARNING;
params->b_annexb = 0;
framebuf = new uint8_t[MAX_FRAMELEN];
bitbuf = new uint8_t[MAX_FRAMELEN];
mux = new Mp4Context;
@@ -57,6 +70,7 @@ TightX264Encoder::TightX264Encoder(SConnection* conn, EncCache *cache_, uint8_t
TightX264Encoder::~TightX264Encoder()
{
delete params;
delete [] framebuf;
delete [] bitbuf;
delete mux;
@@ -95,206 +109,343 @@ void TightX264Encoder::writeRect(const PixelBuffer* pb, const Palette& palette)
w = pb->width();
h = pb->height();
/*w += w & 1;
h += h & 1;*/
if (w != myw || h != myh) {
if (nvidia_init_done)
nvidia_unload();
nvidia_init_done = false;
}
if (!nvidia_init_done) {
if (nvidia_init(w, h, rfb::Server::x264Bitrate,
rfb::Server::frameRate) != 0) {
vlog.error("nvidia init failed, disabling h264");
rfb::Server::x264Bitrate.setParam(0);
return;
}
nvidia_init_done = true;
myw = w;
myh = h;
}
os = conn->getOutStream();
uint32_t cachelen;
const void *cachedata;
if (encCache->enabled &&
(cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) {
os->writeU8(tightX264 << 4);
writeCompact(cachelen, os);
os->writeBytes(cachedata, cachelen);
framectr++;
return;
}
if (using_nvidia) {
if (keyframe) {
framectr = 0;
keyframe = false;
free(mux->buf_header.buf);
free(mux->buf_mdat.buf);
free(mux->buf_moof.buf);
memset(mux, 0, sizeof(Mp4Context));
memset(muxstate, 0, sizeof(Mp4State));
}
mux->framerate = rfb::Server::frameRate;
mux->w = w;
mux->h = h;
buffer = pb->getBuffer(pb->getRect(), &stride);
// Convert it to yuv420 using libwebp's helper functions
/* WebPPicture pic;
WebPPictureInit(&pic);
pic.width = pb->getRect().width();
pic.height = pb->getRect().height();
*/
bool freebuffer = false;
/*if (pic.width & 1 || pic.height & 1) {
// Expand to divisible-by-2 for x264
freebuffer = true;
const uint32_t oldw = pic.width;
const uint32_t oldh = pic.height;
pic.width += pic.width & 1;
pic.height += pic.height & 1;
stride = pic.width;
const rdr::U8 *oldbuffer = buffer;
buffer = (const rdr::U8*) calloc(pic.width * pic.height, 4);
uint32_t y;
for (y = 0; y < oldh; y++)
memcpy((void *) &buffer[y * stride * 4], &oldbuffer[y * oldw * 4], oldw * 4);
}*/
/* if (pfRGBX.equal(pb->getPF())) {
WebPPictureImportRGBX(&pic, buffer, stride * 4);
} else if (pfBGRX.equal(pb->getPF())) {
WebPPictureImportBGRX(&pic, buffer, stride * 4);
} else {
rdr::U8* tmpbuf = new rdr::U8[pic.width * pic.height * 3];
pb->getPF().rgbFromBuffer(tmpbuf, (const rdr::U8 *) buffer, pic.width, stride, pic.height);
stride = pic.width * 3;
WebPPictureImportRGB(&pic, tmpbuf, stride);
delete [] tmpbuf;
}*/
if (!pfBGRX.equal(pb->getPF())) {
vlog.error("unsupported pixel format");
return;
}
if (freebuffer)
free((void *) buffer);
// Encode
uint32_t bitlen;
if (nvenc_frame(buffer, framectr++, bitbuf, bitlen) != 0) {
vlog.error("encoding failed");
return;
}
// Need to parse NALs out of the stream
const uint8_t prefix[3] = { 0, 0, 1 };
const uint8_t *nalptr = bitbuf;
int i_nals = 0;
const uint8_t *nalstarts[32] = { NULL };
uint32_t nallens[32] = { 0 };
uint32_t remlen = bitlen;
while (1) {
const uint8_t *next = (uint8_t *) memmem(nalptr, remlen, prefix, 3);
if (!next)
break;
remlen -= (next + 3) - nalptr;
nalptr = nalstarts[i_nals] = next + 3;
i_nals++;
};
// Lens
int i;
for (i = 0; i < i_nals; i++) {
if (i == i_nals - 1) {
nallens[i] = bitbuf + bitlen - nalstarts[i];
} else {
nallens[i] = nalstarts[i + 1] - nalstarts[i] - 3;
if (w != myw || h != myh) {
if (nvidia_init_done)
nvidia_unload();
nvidia_init_done = false;
}
}
// Mux
framelen = 0;
os->writeU8(tightX264 << 4);
for (i = 0; i < i_nals; i++) {
uint32_t pack_len = nallens[i];
const uint8_t *pack_data = nalstarts[i];
struct NAL nal; nal_parse_header(&nal, pack_data[0]);
switch (nal.unit_type) {
case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; }
case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; }
case NalUnitType_CodedSliceIdr:
case NalUnitType_CodedSliceNonIdr: {
// Write all remaining NALs under the assumption they are the same type.
const uint32_t origlen = pack_len;
pack_len = bitbuf + bitlen - pack_data;
set_slice(mux, pack_data, origlen, pack_len, nal.unit_type);
break;
if (!nvidia_init_done) {
if (nvidia_init(w, h, rfb::Server::x264Bitrate,
rfb::Server::frameRate) != 0) {
vlog.error("nvidia init failed, disabling h264");
rfb::Server::x264Bitrate.setParam(0);
return;
}
default: break;
nvidia_init_done = true;
myw = w;
myh = h;
}
if (nal.unit_type != NalUnitType_CodedSliceIdr &&
nal.unit_type != NalUnitType_CodedSliceNonIdr)
continue;
enum BufError err;
if (!muxstate->header_sent) {
struct BitBuf header_buf;
err = get_header(mux, &header_buf); chk_err_continue
mp4_write_callback(header_buf.buf, header_buf.offset);
muxstate->sequence_number = 1;
muxstate->base_data_offset = header_buf.offset;
muxstate->base_media_decode_time = 0;
muxstate->header_sent = true;
muxstate->nals_count = 0;
muxstate->default_sample_duration = default_sample_size;
uint32_t cachelen;
const void *cachedata;
if (encCache->enabled &&
(cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) {
os->writeU8(tightX264 << 4);
writeCompact(cachelen, os);
os->writeBytes(cachedata, cachelen);
framectr++;
return;
}
err = set_mp4_state(mux, muxstate); chk_err_continue
{
struct BitBuf moof_buf;
err = get_moof(mux, &moof_buf); chk_err_continue
mp4_write_callback(moof_buf.buf, moof_buf.offset);
}
{
struct BitBuf mdat_buf;
err = get_mdat(mux, &mdat_buf); chk_err_continue
mp4_write_callback(mdat_buf.buf, mdat_buf.offset);
if (keyframe) {
framectr = 0;
keyframe = false;
free(mux->buf_header.buf);
free(mux->buf_mdat.buf);
free(mux->buf_moof.buf);
memset(mux, 0, sizeof(Mp4Context));
memset(muxstate, 0, sizeof(Mp4State));
}
break;
mux->framerate = rfb::Server::frameRate;
mux->w = w;
mux->h = h;
buffer = pb->getBuffer(pb->getRect(), &stride);
if (!pfBGRX.equal(pb->getPF())) {
vlog.error("unsupported pixel format");
return;
}
// Encode
uint32_t bitlen;
if (nvenc_frame(buffer, framectr++, bitbuf, bitlen) != 0) {
vlog.error("encoding failed");
return;
}
// Need to parse NALs out of the stream
const uint8_t prefix[3] = { 0, 0, 1 };
const uint8_t *nalptr = bitbuf;
int i_nals = 0;
const uint8_t *nalstarts[32] = { NULL };
uint32_t nallens[32] = { 0 };
uint32_t remlen = bitlen;
while (1) {
const uint8_t *next = (uint8_t *) memmem(nalptr, remlen, prefix, 3);
if (!next)
break;
remlen -= (next + 3) - nalptr;
nalptr = nalstarts[i_nals] = next + 3;
i_nals++;
};
// Lens
int i;
for (i = 0; i < i_nals; i++) {
if (i == i_nals - 1) {
nallens[i] = bitbuf + bitlen - nalstarts[i];
} else {
nallens[i] = nalstarts[i + 1] - nalstarts[i] - 3;
}
}
// Mux
framelen = 0;
os->writeU8(tightX264 << 4);
for (i = 0; i < i_nals; i++) {
uint32_t pack_len = nallens[i];
const uint8_t *pack_data = nalstarts[i];
struct NAL nal; nal_parse_header(&nal, pack_data[0]);
switch (nal.unit_type) {
case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; }
case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; }
case NalUnitType_CodedSliceIdr:
case NalUnitType_CodedSliceNonIdr: {
// Write all remaining NALs under the assumption they are the same type.
const uint32_t origlen = pack_len;
pack_len = bitbuf + bitlen - pack_data;
set_slice(mux, pack_data, origlen, pack_len, nal.unit_type);
break;
}
default: break;
}
if (nal.unit_type != NalUnitType_CodedSliceIdr &&
nal.unit_type != NalUnitType_CodedSliceNonIdr)
continue;
enum BufError err;
if (!muxstate->header_sent) {
struct BitBuf header_buf;
err = get_header(mux, &header_buf); chk_err_continue
mp4_write_callback(header_buf.buf, header_buf.offset);
muxstate->sequence_number = 1;
muxstate->base_data_offset = header_buf.offset;
muxstate->base_media_decode_time = 0;
muxstate->header_sent = true;
muxstate->nals_count = 0;
muxstate->default_sample_duration = default_sample_size;
}
err = set_mp4_state(mux, muxstate); chk_err_continue
{
struct BitBuf moof_buf;
err = get_moof(mux, &moof_buf); chk_err_continue
mp4_write_callback(moof_buf.buf, moof_buf.offset);
}
{
struct BitBuf mdat_buf;
err = get_mdat(mux, &mdat_buf); chk_err_continue
mp4_write_callback(mdat_buf.buf, mdat_buf.offset);
}
break;
}
if (encCache->enabled) {
void *tmp = malloc(framelen);
memcpy(tmp, framebuf, framelen);
encCache->add(cacheType, framectr, 0, w, h, framelen, tmp);
}
writeCompact(framelen, os);
os->writeBytes(framebuf, framelen);
} else {
w += w & 1;
h += h & 1;
params->i_width = w;
params->i_height = h;
x264_param_apply_profile(params, "baseline");
uint32_t cachelen;
const void *cachedata;
if (encCache->enabled &&
(cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) {
os->writeU8(tightX264 << 4);
writeCompact(cachelen, os);
os->writeBytes(cachedata, cachelen);
framectr++;
return;
}
if (keyframe) {
framectr = 0;
keyframe = false;
free(mux->buf_header.buf);
free(mux->buf_mdat.buf);
free(mux->buf_moof.buf);
memset(mux, 0, sizeof(Mp4Context));
memset(muxstate, 0, sizeof(Mp4State));
}
mux->framerate = rfb::Server::frameRate;
mux->w = params->i_width;
mux->h = params->i_height;
if (!enc) {
enc = x264_encoder_open(params);
}
buffer = pb->getBuffer(pb->getRect(), &stride);
// Convert it to yuv420 using libwebp's helper functions
WebPPicture pic;
WebPPictureInit(&pic);
pic.width = pb->getRect().width();
pic.height = pb->getRect().height();
bool freebuffer = false;
if (pic.width & 1 || pic.height & 1) {
// Expand to divisible-by-2 for x264
freebuffer = true;
const uint32_t oldw = pic.width;
const uint32_t oldh = pic.height;
pic.width += pic.width & 1;
pic.height += pic.height & 1;
stride = pic.width;
const rdr::U8 *oldbuffer = buffer;
buffer = (const rdr::U8*) calloc(pic.width * pic.height, 4);
uint32_t y;
for (y = 0; y < oldh; y++)
memcpy((void *) &buffer[y * stride * 4], &oldbuffer[y * oldw * 4], oldw * 4);
}
if (pfRGBX.equal(pb->getPF())) {
WebPPictureImportRGBX(&pic, buffer, stride * 4);
} else if (pfBGRX.equal(pb->getPF())) {
WebPPictureImportBGRX(&pic, buffer, stride * 4);
} else {
rdr::U8* tmpbuf = new rdr::U8[pic.width * pic.height * 3];
pb->getPF().rgbFromBuffer(tmpbuf, (const rdr::U8 *) buffer, pic.width, stride, pic.height);
stride = pic.width * 3;
WebPPictureImportRGB(&pic, tmpbuf, stride);
delete [] tmpbuf;
}
if (freebuffer)
free((void *) buffer);
// Wrap
x264_picture_t pic_in, pic_out;
x264_picture_init(&pic_in);
pic_in.img.i_csp = X264_CSP_I420;
pic_in.img.i_plane = 3;
pic_in.img.plane[0] = pic.y;
pic_in.img.plane[1] = pic.u;
pic_in.img.plane[2] = pic.v;
pic_in.img.i_stride[0] = pic.y_stride;
pic_in.img.i_stride[1] = pic_in.img.i_stride[2] = pic.uv_stride;
pic_in.i_pts = framectr++;
// Encode
int i_nals;
x264_nal_t *nals;
const int len = x264_encoder_encode(enc, &nals, &i_nals, &pic_in, &pic_out);
if (len <= 0 || i_nals <= 0)
vlog.info("encoding error");
// Mux
framelen = 0;
os->writeU8(tightX264 << 4);
int i;
for (i = 0; i < i_nals; i++) {
uint32_t pack_len = nals[i].i_payload - 4;
const uint8_t *pack_data = nals[i].p_payload;
pack_data += 4; // Skip size
struct NAL nal; nal_parse_header(&nal, pack_data[0]);
switch (nal.unit_type) {
case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; }
case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; }
case NalUnitType_CodedSliceIdr:
case NalUnitType_CodedSliceNonIdr: {
// Write all remaining NALs under the assumption they are the same type.
const uint32_t origlen = pack_len;
int j;
for (j = i + 1; j < i_nals; j++)
pack_len += nals[j].i_payload;
set_slice(mux, pack_data, origlen, pack_len, nal.unit_type);
break;
}
default: break;
}
if (nal.unit_type != NalUnitType_CodedSliceIdr &&
nal.unit_type != NalUnitType_CodedSliceNonIdr)
continue;
enum BufError err;
if (!muxstate->header_sent) {
struct BitBuf header_buf;
err = get_header(mux, &header_buf); chk_err_continue
mp4_write_callback(header_buf.buf, header_buf.offset);
muxstate->sequence_number = 1;
muxstate->base_data_offset = header_buf.offset;
muxstate->base_media_decode_time = 0;
muxstate->header_sent = true;
muxstate->nals_count = 0;
muxstate->default_sample_duration = default_sample_size;
}
err = set_mp4_state(mux, muxstate); chk_err_continue
{
struct BitBuf moof_buf;
err = get_moof(mux, &moof_buf); chk_err_continue
mp4_write_callback(moof_buf.buf, moof_buf.offset);
}
{
struct BitBuf mdat_buf;
err = get_mdat(mux, &mdat_buf); chk_err_continue
mp4_write_callback(mdat_buf.buf, mdat_buf.offset);
}
break;
}
if (encCache->enabled) {
void *tmp = malloc(framelen);
memcpy(tmp, framebuf, framelen);
encCache->add(cacheType, framectr, 0, w, h, framelen, tmp);
}
writeCompact(framelen, os);
os->writeBytes(framebuf, framelen);
// Cleanup
WebPPictureFree(&pic);
x264_encoder_close(enc);
enc = NULL;
}
if (encCache->enabled) {
void *tmp = malloc(framelen);
memcpy(tmp, framebuf, framelen);
encCache->add(cacheType, framectr, 0, w, h, framelen, tmp);
}
writeCompact(framelen, os);
os->writeBytes(framebuf, framelen);
// Cleanup
//WebPPictureFree(&pic);
}
void TightX264Encoder::writeSolidRect(int width, int height,
@@ -336,9 +487,12 @@ bool TightX264Encoder::tryInit(const PixelBuffer* pb) {
if (nvidia_init(w, h, rfb::Server::x264Bitrate,
rfb::Server::frameRate) != 0) {
vlog.error("nvidia init failed, disabling h264");
rfb::Server::x264Bitrate.setParam(0);
return false;
vlog.error("nvidia init failed, falling back to x264");
using_nvidia = false;
nvidia_init_done = true;
myw = w;
myh = h;
return true;
}
nvidia_init_done = true;

3
common/rfb/TightX264Encoder.h
View File

@@ -58,11 +58,14 @@ namespace rfb {
protected:
bool keyframe;
x264_t *enc;
x264_param_t *params;
Mp4Context *mux;
Mp4State *muxstate;
unsigned framectr;
bool nvidia_init_done;
bool using_nvidia;
EncCache *encCache;
uint8_t cacheType;

2
unix/xserver/hw/vnc/Makefile.am
View File

@@ -46,7 +46,7 @@ Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \
-I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir)
Xvnc_LDADD = $(XVNC_LIBS) libvnccommon.la $(COMMON_LIBS) \
$(XSERVER_LIBS) $(XSERVER_SYS_LIBS) $(XVNC_SYS_LIBS) -lX11 -lwebp -lssl -lcrypto -lcrypt
$(XSERVER_LIBS) $(XSERVER_SYS_LIBS) $(XVNC_SYS_LIBS) -lX11 -lwebp -lssl -lcrypto -lcrypt -lx264
Xvnc_LDFLAGS = $(LD_EXPORT_SYMBOLS_FLAG) -fopenmp