Resolve KASM-2301 "Feature/ upload debug packages"

3 years ago · 49890cf8da
parent 974a06bb8c
commit 49890cf8da
8 changed files with 129 additions and 15 deletions
--- a/.ci/upload.sh
+++ b/.ci/upload.sh
@ -6,6 +6,27 @@ is_kasmvnc() {
  echo "$package" | grep -q 'kasmvncserver_'
 }

+detect_deb_package_arch() {
+  local deb_package="$1"
+  echo "$deb_package" | sed -e 's/.\+_\([^.]\+\)\.\(d\?\)deb/\1/'
+}
+
+find_deb_package() {
+  local dbgsym_package="$1"
+
+  echo "$dbgsym_package" | sed -e 's/-dbgsym//; s/ddeb/deb/'
+}
+
+fetch_xvnc_md5sum() {
+  local deb_package="$1"
+  deb_package=$(realpath "$deb_package")
+
+  local tmpdir=$(mktemp -d)
+  cd "$tmpdir"
+  dpkg-deb -e "$deb_package"
+  cat DEBIAN/md5sums | grep bin/Xkasmvnc | cut -d' ' -f 1
+}
+
 function prepare_upload_filename() {
  local package="$1";

@ -34,19 +55,18 @@ function prepare_upload_filename() {
 function upload_to_s3() {
  local package="$1";
  local upload_filename="$2";
+  local s3_bucket="$3";

  # Transfer to S3
-  python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${S3_BUCKET}" "$package" "${S3_BUILD_DIRECTORY}/${upload_filename}";
+  python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${s3_bucket}" "$package" "${upload_filename}";
  # Use the Gitlab API to tell Gitlab where the artifact was stored
-  export S3_URL="https://${S3_BUCKET}.s3.amazonaws.com/${S3_BUILD_DIRECTORY}/${upload_filename}";
-  export BUILD_STATUS="{\"key\":\"doc\", \"state\":\"SUCCESSFUL\", \"name\":\"${upload_filename}\", \"url\":\"${S3_URL}\"}";
-  curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=build-url&target_url=${S3_URL}";
+  export S3_URL="https://${s3_bucket}.s3.amazonaws.com/${upload_filename}";
 };

 function prepare_to_run_scripts_and_s3_uploads() {
  export DEBIAN_FRONTEND=noninteractive;
  apt-get update;
-  apt-get install -y ruby2.7 git;
+  apt-get install -y ruby2.7 git wget;
  apt-get install -y python3 python3-pip python3-boto3 curl pkg-config libxmlsec1-dev;
  git clone https://bitbucket.org/awslabs/amazon-s3-bitbucket-pipelines-python.git;
 };
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -3,9 +3,14 @@ services:
  - docker:dind

 variables:
+  KASMVNC_COMMIT_ID: $CI_COMMIT_SHA
  GITLAB_SHARED_DIND_DIR: /builds/$CI_PROJECT_PATH/shared
  GIT_SUBMODULE_STRATEGY: normal
  GIT_FETCH_EXTRA_FLAGS: --tags
+  # E.g. BUILD_JOBS: build_debian_buster,build_ubuntu_bionic. This will include
+  # arm builds, because build_debian_buster_arm matches build_debian_buster.
+  # "BUILD_JOBS: none" won't build any build jobs, nor www.
+  BUILD_JOBS: all

 stages:
  - www
@ -37,6 +42,9 @@ build_www:
    - mkdir -p output/www
    - cd builder
    - tar -zcvf ../output/www/kasm_www.tar.gz www
+  only:
+    variables:
+      - $BUILD_JOBS !~ /^none$/
  artifacts:
    paths:
      - output/
@ -51,6 +59,9 @@ build_ubuntu_bionic:
    - *prepare_artfacts
  script:
    - bash builder/build-package ubuntu bionic
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -67,6 +78,9 @@ build_ubuntu_bionic_arm:
    - *prepare_artfacts
  script:
    - bash builder/build-package ubuntu bionic
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -81,6 +95,9 @@ build_ubuntu_bionic_libjpeg_turbo:
    - *prepare_artfacts
  script:
    - bash builder/build-package ubuntu bionic +libjpeg-turbo_latest
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -95,6 +112,9 @@ build_ubuntu_focal:
    - *prepare_artfacts
  script:
    - bash builder/build-package ubuntu focal;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -111,6 +131,9 @@ build_ubuntu_focal_arm:
    - *prepare_artfacts
  script:
    - bash builder/build-package ubuntu focal;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -127,7 +150,7 @@ build_debian_buster:
    - bash builder/build-package debian buster;
  only:
    variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -146,7 +169,7 @@ build_debian_buster_arm:
    - bash builder/build-package debian buster;
  only:
    variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -163,7 +186,7 @@ build_debian_bullseye:
    - bash builder/build-package debian bullseye;
  only:
    variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -182,7 +205,7 @@ build_debian_bullseye_arm:
    - bash builder/build-package debian bullseye;
  only:
    variables:
-      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
+      - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -197,6 +220,9 @@ build_kali_rolling:
    - *prepare_artfacts
  script:
    - bash builder/build-package kali kali-rolling;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -213,6 +239,9 @@ build_kali_rolling_arm:
    - *prepare_artfacts
  script:
    - bash builder/build-package kali kali-rolling;
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -227,6 +256,9 @@ build_centos7:
    - *prepare_artfacts
  script:
    - bash builder/build-package centos core
+  only:
+    variables:
+      - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
  artifacts:
    paths:
      - output/
@ -237,14 +269,25 @@ upload:
  before_script:
    - . .ci/upload.sh
  script:
-    - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
    - prepare_to_run_scripts_and_s3_uploads
+    - S3_CRASHPAD_BUILD_DIRECTORY="kasmvnc/crashpad/${CI_COMMIT_SHA}"
+    - for dbgsym_package in `find output/ -type f -name '*dbgsym*deb'`; do
+        deb_package=$(find_deb_package "$dbgsym_package");
+        xvnc_md5sum=$(fetch_xvnc_md5sum "$deb_package");
+        upload_filename="${S3_CRASHPAD_BUILD_DIRECTORY}/${xvnc_md5sum}/kasmvncserver-dbgsym.deb";
+        echo;
+        echo "File to upload $upload_filename";
+        upload_to_s3 "$dbgsym_package" "$upload_filename" "$S3_BUCKET";
+        rm "$dbgsym_package";
+      done
+    - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
    - export RELEASE_VERSION=$(.ci/next_release_version "$CI_COMMIT_REF_NAME")
    - for package in `find output/ -type f -name '*.deb' -or -name '*.rpm'`; do
        prepare_upload_filename "$package";
+        upload_filename="${S3_BUILD_DIRECTORY}/$upload_filename";
        echo;
        echo "File to upload $upload_filename";
-        upload_to_s3 "$package" "$upload_filename";
-        UPLOAD_NAME=$(echo $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
+        upload_to_s3 "$package" "$upload_filename" "$S3_BUCKET";
+        UPLOAD_NAME=$(basename $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
        curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=${UPLOAD_NAME}&target_url=${S3_URL}";
      done
--- a/builder/build-deb-inside-docker
+++ b/builder/build-deb-inside-docker
@ -29,5 +29,6 @@ fi

 dpkg-buildpackage -us -uc -b
 mkdir -p "$os_dir"
+cp ../*dbgsym*deb "$os_dir"
 cp ../*.deb "$os_dir"
 lintian ../*.deb || true
--- a/builder/build-tarball
+++ b/builder/build-tarball
@ -31,6 +31,7 @@ docker build -t "$builder_image" \
  -f builder/dockerfile.${os}_${os_codename}${build_tag}.build .
 mkdir -p builder/build
 docker run -v $shared_with_docker_dir:/build -e BUILD_TAG="$build_tag" \
+  -e KASMVNC_COMMIT_ID="$KASMVNC_COMMIT_ID" \
  --rm "$builder_image"

 L_GID=$(id -g)
--- a/builder/build.sh
+++ b/builder/build.sh
@ -7,6 +7,12 @@ detect_quilt() {
  fi
 }

+ensure_crashpad_can_fetch_line_number_by_address() {
+  if [ ! -f /etc/centos-release ]; then
+    export LDFLAGS="$LDFLAGS -no-pie"
+  fi
+}
+
 # For build-dep to work, the apt sources need to have the source server
 #sudo apt-get build-dep xorg-server

@ -38,7 +44,7 @@ tar -C unix/xserver -xf /tmp/xorg-server-${XORG_VER}.tar.bz2 --strip-components=
 cd unix/xserver
 patch -Np1 -i ../xserver${XORG_PATCH}.patch
 case "$XORG_VER" in
-  1.20.*) 
+  1.20.*)
      if [ -f ../xserver120.7.patch ]; then
        patch -Np1 -i ../xserver120.7.patch
      fi ;;
@ -49,12 +55,13 @@ autoreconf -i
 # The distro paths start after prefix and end with the font path,
 # everything after that is based on BUILDING.txt to remove unneeded
 # components.
+ensure_crashpad_can_fetch_line_number_by_address
 ./configure --prefix=/opt/kasmweb \
 	--with-xkb-path=/usr/share/X11/xkb \
 	--with-xkb-output=/var/lib/xkb \
 	--with-xkb-bin-directory=/usr/bin \
 	--with-default-font-path="/usr/share/fonts/X11/misc,/usr/share/fonts/X11/cyrillic,/usr/share/fonts/X11/100dpi/:unscaled,/usr/share/fonts/X11/75dpi/:unscaled,/usr/share/fonts/X11/Type1,/usr/share/fonts/X11/100dpi,/usr/share/fonts/X11/75dpi,built-ins" \
-	--with-pic --without-dtrace --disable-dri \
+	--without-dtrace --disable-dri \
        --disable-static \
 	--disable-xinerama --disable-xvfb --disable-xnest --disable-xorg \
 	--disable-dmx --disable-xwin --disable-xephyr --disable-kdrive \
--- a/unix/kasm-upload-crash-dump.sh
+++ b/unix/kasm-upload-crash-dump.sh
@ -0,0 +1,39 @@
+#!/bin/sh
+
+URL=https://to-be-filled.com/path
+
+die() {
+	echo "$@"
+	exit
+}
+
+[ "$#" -ne 1 ] && die "Usage: $0 logfile"
+
+grep -q Backtrace: $1 || die "No crash in log file"
+
+CURL=`which curl`
+WGET=`which wget`
+
+[ -n "$CURL" -o -n "$WGET" ] || die "Curl or wget required"
+
+BIN=`grep vnc $1 | tail -n1 | cut -d: -f2 | cut -d\( -f1`
+[ -f $BIN ] || die "Can't locate binary"
+
+#
+# prep done, filter the log file
+#
+
+TMP=`mktemp`
+
+LANG=C date >> $TMP
+md5sum $BIN >> $TMP
+$BIN -version 2>&1 | grep built >> $TMP
+grep -A200 Backtrace: $1 >> $TMP
+
+if [ -n "$CURL" ]; then
+	echo curl --data-binary @"$TMP" "$URL"
+else
+	echo wget --post-file "$TMP" "$URL"
+fi
+
+rm $TMP
--- a/unix/xserver/hw/vnc/Makefile.am
+++ b/unix/xserver/hw/vnc/Makefile.am
@ -41,6 +41,7 @@ nodist_EXTRA_Xvnc_SOURCES = dummy.cxx

 Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \
 	-UHAVE_CONFIG_H \
+	-DXVNCEXTRAVERSION="\".$(KASMVNC_COMMIT_ID)\"" \
 	-DXFree86Server -DVENDOR_RELEASE="$(VENDOR_RELEASE)" \
 	-DVENDOR_STRING="\"$(VENDOR_STRING)\"" -I$(KASMVNC_SRCDIR)/common -I$(KASMVNC_SRCDIR)/unix/common \
 	-I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir)
--- a/unix/xserver/hw/vnc/xvnc.c
+++ b/unix/xserver/hw/vnc/xvnc.c
@ -159,7 +159,9 @@ static int vncVerbose = DEFAULT_LOG_VERBOSITY;
 static void
 vncPrintBanner(void)
 {
-    ErrorF("\nXvnc %s - built %s\n%s", XVNCVERSION, buildtime, XVNCCOPYRIGHT);
+    ErrorF("\nXvnc %s%s - built %s\n%s", XVNCVERSION,
+           sizeof(XVNCEXTRAVERSION) > 2 ? XVNCEXTRAVERSION : "",
+           buildtime, XVNCCOPYRIGHT);
    ErrorF("Underlying X server release %d, %s\n\n", VENDOR_RELEASE,
           VENDOR_STRING);
 }