From ac56f5566bc89841e04cfaa4aa793142b06529c8 Mon Sep 17 00:00:00 2001 From: Lauri Kasanen Date: Mon, 22 Nov 2021 11:41:22 +0200 Subject: [PATCH] Restore x264 as a fallback --- common/rfb/TightX264Encoder.cxx | 506 +++++++++++++++++++++----------- common/rfb/TightX264Encoder.h | 3 + unix/xserver/hw/vnc/Makefile.am | 2 +- 3 files changed, 334 insertions(+), 177 deletions(-) diff --git a/common/rfb/TightX264Encoder.cxx b/common/rfb/TightX264Encoder.cxx index 103b2fd..b56b890 100644 --- a/common/rfb/TightX264Encoder.cxx +++ b/common/rfb/TightX264Encoder.cxx @@ -28,6 +28,7 @@ #include #include +#include #include "nvidia.h" #include "mp4.h" @@ -42,11 +43,23 @@ static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0); TightX264Encoder::TightX264Encoder(SConnection* conn, EncCache *cache_, uint8_t cacheType_) : Encoder(conn, encodingTight, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1), - keyframe(true), mux(NULL), muxstate(NULL), framectr(0), - nvidia_init_done(false), + keyframe(true), enc(NULL), params(NULL), mux(NULL), muxstate(NULL), framectr(0), + nvidia_init_done(false), using_nvidia(true), encCache(cache_), cacheType(cacheType_), framebuf(NULL), framelen(0), bitbuf(NULL), myw(0), myh(0) { + params = new x264_param_t; + x264_param_default_preset(params, "veryfast", "zerolatency"); + + params->i_threads = X264_THREADS_AUTO; + params->i_fps_num = params->i_keyint_max = rfb::Server::frameRate; + params->i_fps_den = 1; + params->rc.i_rc_method = X264_RC_ABR; + params->rc.i_bitrate = rfb::Server::x264Bitrate; + params->i_csp = X264_CSP_I420; + params->i_log_level = X264_LOG_WARNING; + params->b_annexb = 0; + framebuf = new uint8_t[MAX_FRAMELEN]; bitbuf = new uint8_t[MAX_FRAMELEN]; mux = new Mp4Context; @@ -57,6 +70,7 @@ TightX264Encoder::TightX264Encoder(SConnection* conn, EncCache *cache_, uint8_t TightX264Encoder::~TightX264Encoder() { + delete params; delete [] framebuf; delete [] bitbuf; delete mux; @@ -95,206 +109,343 @@ void TightX264Encoder::writeRect(const PixelBuffer* pb, const Palette& palette) w = pb->width(); h = pb->height(); - /*w += w & 1; - h += h & 1;*/ + os = conn->getOutStream(); - if (w != myw || h != myh) { - if (nvidia_init_done) - nvidia_unload(); - nvidia_init_done = false; - } + if (using_nvidia) { + + if (w != myw || h != myh) { + if (nvidia_init_done) + nvidia_unload(); + nvidia_init_done = false; + } - if (!nvidia_init_done) { - if (nvidia_init(w, h, rfb::Server::x264Bitrate, - rfb::Server::frameRate) != 0) { - vlog.error("nvidia init failed, disabling h264"); - rfb::Server::x264Bitrate.setParam(0); + if (!nvidia_init_done) { + if (nvidia_init(w, h, rfb::Server::x264Bitrate, + rfb::Server::frameRate) != 0) { + vlog.error("nvidia init failed, disabling h264"); + rfb::Server::x264Bitrate.setParam(0); + return; + } + nvidia_init_done = true; + myw = w; + myh = h; + } + + uint32_t cachelen; + const void *cachedata; + if (encCache->enabled && + (cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) { + os->writeU8(tightX264 << 4); + writeCompact(cachelen, os); + os->writeBytes(cachedata, cachelen); + framectr++; return; } - nvidia_init_done = true; - myw = w; - myh = h; - } - os = conn->getOutStream(); + if (keyframe) { + framectr = 0; + keyframe = false; - uint32_t cachelen; - const void *cachedata; - if (encCache->enabled && - (cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) { - os->writeU8(tightX264 << 4); - writeCompact(cachelen, os); - os->writeBytes(cachedata, cachelen); - framectr++; - return; - } + free(mux->buf_header.buf); + free(mux->buf_mdat.buf); + free(mux->buf_moof.buf); + memset(mux, 0, sizeof(Mp4Context)); + memset(muxstate, 0, sizeof(Mp4State)); + } - if (keyframe) { - framectr = 0; - keyframe = false; + mux->framerate = rfb::Server::frameRate; + mux->w = w; + mux->h = h; - free(mux->buf_header.buf); - free(mux->buf_mdat.buf); - free(mux->buf_moof.buf); - memset(mux, 0, sizeof(Mp4Context)); - memset(muxstate, 0, sizeof(Mp4State)); - } + buffer = pb->getBuffer(pb->getRect(), &stride); - mux->framerate = rfb::Server::frameRate; - mux->w = w; - mux->h = h; - - buffer = pb->getBuffer(pb->getRect(), &stride); - - // Convert it to yuv420 using libwebp's helper functions -/* WebPPicture pic; - - WebPPictureInit(&pic); - pic.width = pb->getRect().width(); - pic.height = pb->getRect().height(); -*/ - bool freebuffer = false; - /*if (pic.width & 1 || pic.height & 1) { - // Expand to divisible-by-2 for x264 - freebuffer = true; - const uint32_t oldw = pic.width; - const uint32_t oldh = pic.height; - pic.width += pic.width & 1; - pic.height += pic.height & 1; - stride = pic.width; - const rdr::U8 *oldbuffer = buffer; - buffer = (const rdr::U8*) calloc(pic.width * pic.height, 4); - - uint32_t y; - for (y = 0; y < oldh; y++) - memcpy((void *) &buffer[y * stride * 4], &oldbuffer[y * oldw * 4], oldw * 4); - }*/ - -/* if (pfRGBX.equal(pb->getPF())) { - WebPPictureImportRGBX(&pic, buffer, stride * 4); - } else if (pfBGRX.equal(pb->getPF())) { - WebPPictureImportBGRX(&pic, buffer, stride * 4); - } else { - rdr::U8* tmpbuf = new rdr::U8[pic.width * pic.height * 3]; - pb->getPF().rgbFromBuffer(tmpbuf, (const rdr::U8 *) buffer, pic.width, stride, pic.height); - stride = pic.width * 3; + if (!pfBGRX.equal(pb->getPF())) { + vlog.error("unsupported pixel format"); + return; + } - WebPPictureImportRGB(&pic, tmpbuf, stride); - delete [] tmpbuf; - }*/ + // Encode + uint32_t bitlen; + if (nvenc_frame(buffer, framectr++, bitbuf, bitlen) != 0) { + vlog.error("encoding failed"); + return; + } - if (!pfBGRX.equal(pb->getPF())) { - vlog.error("unsupported pixel format"); - return; - } + // Need to parse NALs out of the stream + const uint8_t prefix[3] = { 0, 0, 1 }; + const uint8_t *nalptr = bitbuf; + int i_nals = 0; + const uint8_t *nalstarts[32] = { NULL }; + uint32_t nallens[32] = { 0 }; + uint32_t remlen = bitlen; + + while (1) { + const uint8_t *next = (uint8_t *) memmem(nalptr, remlen, prefix, 3); + if (!next) + break; + + remlen -= (next + 3) - nalptr; + nalptr = nalstarts[i_nals] = next + 3; + + i_nals++; + }; + + // Lens + int i; + for (i = 0; i < i_nals; i++) { + if (i == i_nals - 1) { + nallens[i] = bitbuf + bitlen - nalstarts[i]; + } else { + nallens[i] = nalstarts[i + 1] - nalstarts[i] - 3; + } + } - if (freebuffer) - free((void *) buffer); + // Mux + framelen = 0; + os->writeU8(tightX264 << 4); - // Encode - uint32_t bitlen; - if (nvenc_frame(buffer, framectr++, bitbuf, bitlen) != 0) { - vlog.error("encoding failed"); - return; - } + for (i = 0; i < i_nals; i++) { + uint32_t pack_len = nallens[i]; + const uint8_t *pack_data = nalstarts[i]; + + struct NAL nal; nal_parse_header(&nal, pack_data[0]); + + switch (nal.unit_type) { + case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; } + case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; } + case NalUnitType_CodedSliceIdr: + case NalUnitType_CodedSliceNonIdr: { + // Write all remaining NALs under the assumption they are the same type. + const uint32_t origlen = pack_len; + pack_len = bitbuf + bitlen - pack_data; + set_slice(mux, pack_data, origlen, pack_len, nal.unit_type); + break; + } + default: break; + } - // Need to parse NALs out of the stream - const uint8_t prefix[3] = { 0, 0, 1 }; - const uint8_t *nalptr = bitbuf; - int i_nals = 0; - const uint8_t *nalstarts[32] = { NULL }; - uint32_t nallens[32] = { 0 }; - uint32_t remlen = bitlen; - - while (1) { - const uint8_t *next = (uint8_t *) memmem(nalptr, remlen, prefix, 3); - if (!next) - break; + if (nal.unit_type != NalUnitType_CodedSliceIdr && + nal.unit_type != NalUnitType_CodedSliceNonIdr) + continue; + + enum BufError err; + if (!muxstate->header_sent) { + struct BitBuf header_buf; + err = get_header(mux, &header_buf); chk_err_continue + mp4_write_callback(header_buf.buf, header_buf.offset); + + muxstate->sequence_number = 1; + muxstate->base_data_offset = header_buf.offset; + muxstate->base_media_decode_time = 0; + muxstate->header_sent = true; + muxstate->nals_count = 0; + muxstate->default_sample_duration = default_sample_size; + } - remlen -= (next + 3) - nalptr; - nalptr = nalstarts[i_nals] = next + 3; + err = set_mp4_state(mux, muxstate); chk_err_continue + { + struct BitBuf moof_buf; + err = get_moof(mux, &moof_buf); chk_err_continue + mp4_write_callback(moof_buf.buf, moof_buf.offset); + } + { + struct BitBuf mdat_buf; + err = get_mdat(mux, &mdat_buf); chk_err_continue + mp4_write_callback(mdat_buf.buf, mdat_buf.offset); + } - i_nals++; - }; + break; + } - // Lens - int i; - for (i = 0; i < i_nals; i++) { - if (i == i_nals - 1) { - nallens[i] = bitbuf + bitlen - nalstarts[i]; - } else { - nallens[i] = nalstarts[i + 1] - nalstarts[i] - 3; + if (encCache->enabled) { + void *tmp = malloc(framelen); + memcpy(tmp, framebuf, framelen); + encCache->add(cacheType, framectr, 0, w, h, framelen, tmp); } - } - // Mux - framelen = 0; - os->writeU8(tightX264 << 4); - - for (i = 0; i < i_nals; i++) { - uint32_t pack_len = nallens[i]; - const uint8_t *pack_data = nalstarts[i]; - - struct NAL nal; nal_parse_header(&nal, pack_data[0]); - - switch (nal.unit_type) { - case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; } - case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; } - case NalUnitType_CodedSliceIdr: - case NalUnitType_CodedSliceNonIdr: { - // Write all remaining NALs under the assumption they are the same type. - const uint32_t origlen = pack_len; - pack_len = bitbuf + bitlen - pack_data; - set_slice(mux, pack_data, origlen, pack_len, nal.unit_type); - break; - } - default: break; + writeCompact(framelen, os); + os->writeBytes(framebuf, framelen); + } else { + w += w & 1; + h += h & 1; + + params->i_width = w; + params->i_height = h; + + x264_param_apply_profile(params, "baseline"); + + uint32_t cachelen; + const void *cachedata; + if (encCache->enabled && + (cachedata = encCache->get(cacheType, framectr, 0, w, h, cachelen))) { + os->writeU8(tightX264 << 4); + writeCompact(cachelen, os); + os->writeBytes(cachedata, cachelen); + framectr++; + return; } - if (nal.unit_type != NalUnitType_CodedSliceIdr && - nal.unit_type != NalUnitType_CodedSliceNonIdr) - continue; - - enum BufError err; - if (!muxstate->header_sent) { - struct BitBuf header_buf; - err = get_header(mux, &header_buf); chk_err_continue - mp4_write_callback(header_buf.buf, header_buf.offset); - - muxstate->sequence_number = 1; - muxstate->base_data_offset = header_buf.offset; - muxstate->base_media_decode_time = 0; - muxstate->header_sent = true; - muxstate->nals_count = 0; - muxstate->default_sample_duration = default_sample_size; + if (keyframe) { + framectr = 0; + keyframe = false; + + free(mux->buf_header.buf); + free(mux->buf_mdat.buf); + free(mux->buf_moof.buf); + memset(mux, 0, sizeof(Mp4Context)); + memset(muxstate, 0, sizeof(Mp4State)); } - err = set_mp4_state(mux, muxstate); chk_err_continue - { - struct BitBuf moof_buf; - err = get_moof(mux, &moof_buf); chk_err_continue - mp4_write_callback(moof_buf.buf, moof_buf.offset); + mux->framerate = rfb::Server::frameRate; + mux->w = params->i_width; + mux->h = params->i_height; + + if (!enc) { + enc = x264_encoder_open(params); } - { - struct BitBuf mdat_buf; - err = get_mdat(mux, &mdat_buf); chk_err_continue - mp4_write_callback(mdat_buf.buf, mdat_buf.offset); + + buffer = pb->getBuffer(pb->getRect(), &stride); + + // Convert it to yuv420 using libwebp's helper functions + WebPPicture pic; + + WebPPictureInit(&pic); + pic.width = pb->getRect().width(); + pic.height = pb->getRect().height(); + + bool freebuffer = false; + if (pic.width & 1 || pic.height & 1) { + // Expand to divisible-by-2 for x264 + freebuffer = true; + const uint32_t oldw = pic.width; + const uint32_t oldh = pic.height; + pic.width += pic.width & 1; + pic.height += pic.height & 1; + stride = pic.width; + const rdr::U8 *oldbuffer = buffer; + buffer = (const rdr::U8*) calloc(pic.width * pic.height, 4); + + uint32_t y; + for (y = 0; y < oldh; y++) + memcpy((void *) &buffer[y * stride * 4], &oldbuffer[y * oldw * 4], oldw * 4); } - break; - } + if (pfRGBX.equal(pb->getPF())) { + WebPPictureImportRGBX(&pic, buffer, stride * 4); + } else if (pfBGRX.equal(pb->getPF())) { + WebPPictureImportBGRX(&pic, buffer, stride * 4); + } else { + rdr::U8* tmpbuf = new rdr::U8[pic.width * pic.height * 3]; + pb->getPF().rgbFromBuffer(tmpbuf, (const rdr::U8 *) buffer, pic.width, stride, pic.height); + stride = pic.width * 3; - if (encCache->enabled) { - void *tmp = malloc(framelen); - memcpy(tmp, framebuf, framelen); - encCache->add(cacheType, framectr, 0, w, h, framelen, tmp); - } + WebPPictureImportRGB(&pic, tmpbuf, stride); + delete [] tmpbuf; + } + + if (freebuffer) + free((void *) buffer); - writeCompact(framelen, os); - os->writeBytes(framebuf, framelen); + // Wrap + x264_picture_t pic_in, pic_out; + x264_picture_init(&pic_in); - // Cleanup - //WebPPictureFree(&pic); + pic_in.img.i_csp = X264_CSP_I420; + pic_in.img.i_plane = 3; + + pic_in.img.plane[0] = pic.y; + pic_in.img.plane[1] = pic.u; + pic_in.img.plane[2] = pic.v; + + pic_in.img.i_stride[0] = pic.y_stride; + pic_in.img.i_stride[1] = pic_in.img.i_stride[2] = pic.uv_stride; + + pic_in.i_pts = framectr++; + + // Encode + int i_nals; + x264_nal_t *nals; + const int len = x264_encoder_encode(enc, &nals, &i_nals, &pic_in, &pic_out); + + if (len <= 0 || i_nals <= 0) + vlog.info("encoding error"); + + // Mux + framelen = 0; + os->writeU8(tightX264 << 4); + + int i; + for (i = 0; i < i_nals; i++) { + uint32_t pack_len = nals[i].i_payload - 4; + const uint8_t *pack_data = nals[i].p_payload; + + pack_data += 4; // Skip size + + struct NAL nal; nal_parse_header(&nal, pack_data[0]); + + switch (nal.unit_type) { + case NalUnitType_SPS: { set_sps(mux, pack_data, pack_len); break; } + case NalUnitType_PPS: { set_pps(mux, pack_data, pack_len); break; } + case NalUnitType_CodedSliceIdr: + case NalUnitType_CodedSliceNonIdr: { + // Write all remaining NALs under the assumption they are the same type. + const uint32_t origlen = pack_len; + int j; + for (j = i + 1; j < i_nals; j++) + pack_len += nals[j].i_payload; + set_slice(mux, pack_data, origlen, pack_len, nal.unit_type); + break; + } + default: break; + } + + if (nal.unit_type != NalUnitType_CodedSliceIdr && + nal.unit_type != NalUnitType_CodedSliceNonIdr) + continue; + + enum BufError err; + if (!muxstate->header_sent) { + struct BitBuf header_buf; + err = get_header(mux, &header_buf); chk_err_continue + mp4_write_callback(header_buf.buf, header_buf.offset); + + muxstate->sequence_number = 1; + muxstate->base_data_offset = header_buf.offset; + muxstate->base_media_decode_time = 0; + muxstate->header_sent = true; + muxstate->nals_count = 0; + muxstate->default_sample_duration = default_sample_size; + } + + err = set_mp4_state(mux, muxstate); chk_err_continue + { + struct BitBuf moof_buf; + err = get_moof(mux, &moof_buf); chk_err_continue + mp4_write_callback(moof_buf.buf, moof_buf.offset); + } + { + struct BitBuf mdat_buf; + err = get_mdat(mux, &mdat_buf); chk_err_continue + mp4_write_callback(mdat_buf.buf, mdat_buf.offset); + } + + break; + } + + if (encCache->enabled) { + void *tmp = malloc(framelen); + memcpy(tmp, framebuf, framelen); + encCache->add(cacheType, framectr, 0, w, h, framelen, tmp); + } + + writeCompact(framelen, os); + os->writeBytes(framebuf, framelen); + + // Cleanup + WebPPictureFree(&pic); + x264_encoder_close(enc); + enc = NULL; + } } void TightX264Encoder::writeSolidRect(int width, int height, @@ -336,9 +487,12 @@ bool TightX264Encoder::tryInit(const PixelBuffer* pb) { if (nvidia_init(w, h, rfb::Server::x264Bitrate, rfb::Server::frameRate) != 0) { - vlog.error("nvidia init failed, disabling h264"); - rfb::Server::x264Bitrate.setParam(0); - return false; + vlog.error("nvidia init failed, falling back to x264"); + using_nvidia = false; + nvidia_init_done = true; + myw = w; + myh = h; + return true; } nvidia_init_done = true; diff --git a/common/rfb/TightX264Encoder.h b/common/rfb/TightX264Encoder.h index 3d7d3c3..c87644a 100644 --- a/common/rfb/TightX264Encoder.h +++ b/common/rfb/TightX264Encoder.h @@ -58,11 +58,14 @@ namespace rfb { protected: bool keyframe; + x264_t *enc; + x264_param_t *params; Mp4Context *mux; Mp4State *muxstate; unsigned framectr; bool nvidia_init_done; + bool using_nvidia; EncCache *encCache; uint8_t cacheType; diff --git a/unix/xserver/hw/vnc/Makefile.am b/unix/xserver/hw/vnc/Makefile.am index 3692555..e3a5ced 100644 --- a/unix/xserver/hw/vnc/Makefile.am +++ b/unix/xserver/hw/vnc/Makefile.am @@ -46,7 +46,7 @@ Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \ -I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir) Xvnc_LDADD = $(XVNC_LIBS) libvnccommon.la $(COMMON_LIBS) \ - $(XSERVER_LIBS) $(XSERVER_SYS_LIBS) $(XVNC_SYS_LIBS) -lX11 -lwebp -lssl -lcrypto -lcrypt + $(XSERVER_LIBS) $(XSERVER_SYS_LIBS) $(XVNC_SYS_LIBS) -lX11 -lwebp -lssl -lcrypto -lcrypt -lx264 Xvnc_LDFLAGS = $(LD_EXPORT_SYMBOLS_FLAG) -fopenmp