Nvidia resizing, init failure check

common/rfb/EncodeManager.cxx

@@ -401,7 +401,8 @@ void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
     /*
      * In extra-low-quality mode, if x264 is enabled, send entire screen frames
      */
-    if (rfb::Server::x264Bitrate && videoDetected) {
+    if (rfb::Server::x264Bitrate && videoDetected &&
+        ((TightX264Encoder *) encoders[encoderTightX264])->tryInit(pb)) {
       std::vector<Rect> rects;
       changed.get_rects(&rects);
       updateVideoStats(rects, pb);

common/rfb/TightX264Encoder.cxx

@@ -45,7 +45,7 @@ TightX264Encoder::TightX264Encoder(SConnection* conn, EncCache *cache_, uint8_t
   keyframe(true), mux(NULL), muxstate(NULL), framectr(0),
   nvidia_init_done(false),
   encCache(cache_), cacheType(cacheType_),
-  framebuf(NULL), framelen(0), bitbuf(NULL)
+  framebuf(NULL), framelen(0), bitbuf(NULL), myw(0), myh(0)
 {
   framebuf = new uint8_t[MAX_FRAMELEN];
   bitbuf = new uint8_t[MAX_FRAMELEN];
@@ -98,12 +98,22 @@ void TightX264Encoder::writeRect(const PixelBuffer* pb, const Palette& palette)
   /*w += w & 1;
   h += h & 1;*/
 
+  if (w != myw || h != myh) {
+    if (nvidia_init_done)
+      nvidia_unload();
+    nvidia_init_done = false;
+  }
+
   if (!nvidia_init_done) {
     if (nvidia_init(w, h, rfb::Server::x264Bitrate,
                 rfb::Server::frameRate) != 0) {
-      vlog.error("nvidia init failed");
+      vlog.error("nvidia init failed, disabling h264");
+      rfb::Server::x264Bitrate.setParam(0);
+      return;
     }
     nvidia_init_done = true;
+    myw = w;
+    myh = h;
   }
 
   os = conn->getOutStream();
@@ -315,3 +325,23 @@ void TightX264Encoder::writeCompact(rdr::U32 value, rdr::OutStream* os) const
     }
   }
 }
+
+bool TightX264Encoder::tryInit(const PixelBuffer* pb) {
+  if (nvidia_init_done)
+    return true;
+
+  uint32_t w, h;
+  w = pb->width();
+  h = pb->height();
+
+  if (nvidia_init(w, h, rfb::Server::x264Bitrate,
+              rfb::Server::frameRate) != 0) {
+    vlog.error("nvidia init failed, disabling h264");
+    rfb::Server::x264Bitrate.setParam(0);
+    return false;
+  }
+
+  nvidia_init_done = true;
+  myw = w;
+  myh = h;
+}

common/rfb/TightX264Encoder.h

@@ -50,6 +50,8 @@ namespace rfb {
 
     virtual void setKeyframe() { keyframe = true; }
 
+    bool tryInit(const PixelBuffer* pb);
+
   protected:
     void writeCompact(rdr::U32 value, rdr::OutStream* os) const;
     void mp4_write_callback(const void *buffer, size_t size);
@@ -69,6 +71,8 @@ namespace rfb {
     uint32_t framelen;
 
     uint8_t *bitbuf;
+
+    uint16_t myw, myh;
   };
 }
 #endif

common/rfb/nvidia.cxx

@@ -387,7 +387,7 @@ int nvenc_frame(const uint8_t *data, unsigned pts, uint8_t *out, uint32_t &outle
 	return 0;
 }
 
-static void unload() {
+void nvidia_unload() {
 	NV_ENC_PIC_PARAMS params;
 	memset(&params, 0, sizeof(NV_ENC_PIC_PARAMS));
 	params.version = NV_ENC_PIC_PARAMS_VER;

common/rfb/nvidia.h

@@ -6,5 +6,6 @@
 int nvidia_init(const unsigned w, const unsigned h, const unsigned kbps,
 		const unsigned fps);
 int nvenc_frame(const uint8_t *data, unsigned pts, uint8_t *out, uint32_t &outlen);
+void nvidia_unload();
 
 #endif