Merge branch 'feature/KASM-2301-upload-debug-packages' into 'master'

Resolve KASM-2301 "Feature/ upload debug packages"

Closes KASM-2301

See merge request kasm-technologies/internal/KasmVNC!26

.ci/upload.sh

@@ -6,6 +6,27 @@ is_kasmvnc() {
   echo "$package" | grep -q 'kasmvncserver_'
 }
 
+detect_deb_package_arch() {
+  local deb_package="$1"
+  echo "$deb_package" | sed -e 's/.\+_\([^.]\+\)\.\(d\?\)deb/\1/'
+}
+
+find_deb_package() {
+  local dbgsym_package="$1"
+
+  echo "$dbgsym_package" | sed -e 's/-dbgsym//; s/ddeb/deb/'
+}
+
+fetch_xvnc_md5sum() {
+  local deb_package="$1"
+  deb_package=$(realpath "$deb_package")
+
+  local tmpdir=$(mktemp -d)
+  cd "$tmpdir"
+  dpkg-deb -e "$deb_package"
+  cat DEBIAN/md5sums | grep bin/Xkasmvnc | cut -d' ' -f 1
+}
+
 function prepare_upload_filename() {
   local package="$1";
 
@@ -34,19 +55,18 @@ function prepare_upload_filename() {
 function upload_to_s3() {
   local package="$1";
   local upload_filename="$2";
+  local s3_bucket="$3";
 
   # Transfer to S3
-  python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${S3_BUCKET}" "$package" "${S3_BUILD_DIRECTORY}/${upload_filename}";
+  python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${s3_bucket}" "$package" "${upload_filename}";
   # Use the Gitlab API to tell Gitlab where the artifact was stored
-  export S3_URL="https://${S3_BUCKET}.s3.amazonaws.com/${S3_BUILD_DIRECTORY}/${upload_filename}";
+  export S3_URL="https://${s3_bucket}.s3.amazonaws.com/${upload_filename}";
-  export BUILD_STATUS="{\"key\":\"doc\", \"state\":\"SUCCESSFUL\", \"name\":\"${upload_filename}\", \"url\":\"${S3_URL}\"}";
-  curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=build-url&target_url=${S3_URL}";
 };
 
 function prepare_to_run_scripts_and_s3_uploads() {
   export DEBIAN_FRONTEND=noninteractive;
   apt-get update;
-  apt-get install -y ruby2.7 git;
+  apt-get install -y ruby2.7 git wget;
   apt-get install -y python3 python3-pip python3-boto3 curl pkg-config libxmlsec1-dev;
   git clone https://bitbucket.org/awslabs/amazon-s3-bitbucket-pipelines-python.git;
 };

.gitlab-ci.yml

@@ -3,9 +3,14 @@ services:
   - docker:dind
 
 variables:
+  KASMVNC_COMMIT_ID: $CI_COMMIT_SHA
   GITLAB_SHARED_DIND_DIR: /builds/$CI_PROJECT_PATH/shared
   GIT_SUBMODULE_STRATEGY: normal
   GIT_FETCH_EXTRA_FLAGS: --tags
+  # E.g. BUILD_JOBS: build_debian_buster,build_ubuntu_bionic. This will include
+  # arm builds, because build_debian_buster_arm matches build_debian_buster.
+  # "BUILD_JOBS: none" won't build any build jobs, nor www.
+  BUILD_JOBS: all
 
 stages:
   - www
@@ -37,6 +42,9 @@ build_www:
     - mkdir -p output/www
     - cd builder
     - tar -zcvf ../output/www/kasm_www.tar.gz www
+  only:
+    variables:
+      - $BUILD_JOBS !~ /^none$/
   artifacts:
     paths:
       - output/
@@ -51,6 +59,9 @@ build_ubuntu_bionic:
     - *prepare_artfacts
   script:
     - bash builder/build-package ubuntu bionic
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -67,6 +78,9 @@ build_ubuntu_bionic_arm:
     - *prepare_artfacts
   script:
     - bash builder/build-package ubuntu bionic
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -81,6 +95,9 @@ build_ubuntu_bionic_libjpeg_turbo:
     - *prepare_artfacts
   script:
     - bash builder/build-package ubuntu bionic +libjpeg-turbo_latest
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -95,6 +112,9 @@ build_ubuntu_focal:
     - *prepare_artfacts
   script:
     - bash builder/build-package ubuntu focal;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -111,6 +131,9 @@ build_ubuntu_focal_arm:
     - *prepare_artfacts
   script:
     - bash builder/build-package ubuntu focal;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -127,7 +150,7 @@ build_debian_buster:
     - bash builder/build-package debian buster;
   only:
     variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -146,7 +169,7 @@ build_debian_buster_arm:
     - bash builder/build-package debian buster;
   only:
     variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -163,7 +186,7 @@ build_debian_bullseye:
     - bash builder/build-package debian bullseye;
   only:
     variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -182,7 +205,7 @@ build_debian_bullseye_arm:
     - bash builder/build-package debian bullseye;
   only:
     variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -197,6 +220,9 @@ build_kali_rolling:
     - *prepare_artfacts
   script:
     - bash builder/build-package kali kali-rolling;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -213,6 +239,9 @@ build_kali_rolling_arm:
     - *prepare_artfacts
   script:
     - bash builder/build-package kali kali-rolling;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -227,6 +256,9 @@ build_centos7:
     - *prepare_artfacts
   script:
     - bash builder/build-package centos core
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
   artifacts:
     paths:
       - output/
@@ -237,14 +269,25 @@ upload:
   before_script:
     - . .ci/upload.sh
   script:
-    - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
     - prepare_to_run_scripts_and_s3_uploads
+    - S3_CRASHPAD_BUILD_DIRECTORY="kasmvnc/crashpad/${CI_COMMIT_SHA}"
+    - for dbgsym_package in `find output/ -type f -name '*dbgsym*deb'`; do
+        deb_package=$(find_deb_package "$dbgsym_package");
+        xvnc_md5sum=$(fetch_xvnc_md5sum "$deb_package");
+        upload_filename="${S3_CRASHPAD_BUILD_DIRECTORY}/${xvnc_md5sum}/kasmvncserver-dbgsym.deb";
+        echo;
+        echo "File to upload $upload_filename";
+        upload_to_s3 "$dbgsym_package" "$upload_filename" "$S3_BUCKET";
+        rm "$dbgsym_package";
+      done
+    - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
     - export RELEASE_VERSION=$(.ci/next_release_version "$CI_COMMIT_REF_NAME")
     - for package in `find output/ -type f -name '*.deb' -or -name '*.rpm'`; do
         prepare_upload_filename "$package";
+        upload_filename="${S3_BUILD_DIRECTORY}/$upload_filename";
         echo;
         echo "File to upload $upload_filename";
-        upload_to_s3 "$package" "$upload_filename";
+        upload_to_s3 "$package" "$upload_filename" "$S3_BUCKET";
-        UPLOAD_NAME=$(echo $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
+        UPLOAD_NAME=$(basename $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
         curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=${UPLOAD_NAME}&target_url=${S3_URL}";
       done

builder/build-deb-inside-docker

@@ -29,5 +29,6 @@ fi
 
 dpkg-buildpackage -us -uc -b
 mkdir -p "$os_dir"
+cp ../*dbgsym*deb "$os_dir"
 cp ../*.deb "$os_dir"
 lintian ../*.deb || true

builder/build-tarball

@@ -31,6 +31,7 @@ docker build -t "$builder_image" \
   -f builder/dockerfile.${os}_${os_codename}${build_tag}.build .
 mkdir -p builder/build
 docker run -v $shared_with_docker_dir:/build -e BUILD_TAG="$build_tag" \
+  -e KASMVNC_COMMIT_ID="$KASMVNC_COMMIT_ID" \
   --rm "$builder_image"
 
 L_GID=$(id -g)

builder/build.sh

@@ -7,6 +7,12 @@ detect_quilt() {
   fi
 }
 
+ensure_crashpad_can_fetch_line_number_by_address() {
+  if [ ! -f /etc/centos-release ]; then
+    export LDFLAGS="$LDFLAGS -no-pie"
+  fi
+}
+
 # For build-dep to work, the apt sources need to have the source server
 #sudo apt-get build-dep xorg-server
 
@@ -49,12 +55,13 @@ autoreconf -i
 # The distro paths start after prefix and end with the font path,
 # everything after that is based on BUILDING.txt to remove unneeded
 # components.
+ensure_crashpad_can_fetch_line_number_by_address
 ./configure --prefix=/opt/kasmweb \
 	--with-xkb-path=/usr/share/X11/xkb \
 	--with-xkb-output=/var/lib/xkb \
 	--with-xkb-bin-directory=/usr/bin \
 	--with-default-font-path="/usr/share/fonts/X11/misc,/usr/share/fonts/X11/cyrillic,/usr/share/fonts/X11/100dpi/:unscaled,/usr/share/fonts/X11/75dpi/:unscaled,/usr/share/fonts/X11/Type1,/usr/share/fonts/X11/100dpi,/usr/share/fonts/X11/75dpi,built-ins" \
-	--with-pic --without-dtrace --disable-dri \
+	--without-dtrace --disable-dri \
         --disable-static \
 	--disable-xinerama --disable-xvfb --disable-xnest --disable-xorg \
 	--disable-dmx --disable-xwin --disable-xephyr --disable-kdrive \

unix/kasm-upload-crash-dump.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+URL=https://to-be-filled.com/path
+
+die() {
+	echo "$@"
+	exit
+}
+
+[ "$#" -ne 1 ] && die "Usage: $0 logfile"
+
+grep -q Backtrace: $1 || die "No crash in log file"
+
+CURL=`which curl`
+WGET=`which wget`
+
+[ -n "$CURL" -o -n "$WGET" ] || die "Curl or wget required"
+
+BIN=`grep vnc $1 | tail -n1 | cut -d: -f2 | cut -d\( -f1`
+[ -f $BIN ] || die "Can't locate binary"
+
+#
+# prep done, filter the log file
+#
+
+TMP=`mktemp`
+
+LANG=C date >> $TMP
+md5sum $BIN >> $TMP
+$BIN -version 2>&1 | grep built >> $TMP
+grep -A200 Backtrace: $1 >> $TMP
+
+if [ -n "$CURL" ]; then
+	echo curl --data-binary @"$TMP" "$URL"
+else
+	echo wget --post-file "$TMP" "$URL"
+fi
+
+rm $TMP

unix/xserver/hw/vnc/Makefile.am

@@ -41,6 +41,7 @@ nodist_EXTRA_Xvnc_SOURCES = dummy.cxx
 
 Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \
 	-UHAVE_CONFIG_H \
+	-DXVNCEXTRAVERSION="\".$(KASMVNC_COMMIT_ID)\"" \
 	-DXFree86Server -DVENDOR_RELEASE="$(VENDOR_RELEASE)" \
 	-DVENDOR_STRING="\"$(VENDOR_STRING)\"" -I$(KASMVNC_SRCDIR)/common -I$(KASMVNC_SRCDIR)/unix/common \
 	-I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir)

unix/xserver/hw/vnc/xvnc.c

@@ -159,7 +159,9 @@ static int vncVerbose = DEFAULT_LOG_VERBOSITY;
 static void
 vncPrintBanner(void)
 {
-    ErrorF("\nXvnc %s - built %s\n%s", XVNCVERSION, buildtime, XVNCCOPYRIGHT);
+    ErrorF("\nXvnc %s%s - built %s\n%s", XVNCVERSION,
+           sizeof(XVNCEXTRAVERSION) > 2 ? XVNCEXTRAVERSION : "",
+           buildtime, XVNCCOPYRIGHT);
     ErrorF("Underlying X server release %d, %s\n\n", VENDOR_RELEASE,
            VENDOR_STRING);
 }