Initial commit of lyx-tester

These scripts help with building and testing LyX, mostly with the
ctest framework.

"lyxbuild" is a build script that has different options (e.g., to
compile with Clang/GCC, Qt 5/6, CMake/autotools). The build script
also has an option to cherry-pick compiler fixes which make it
easier to build older commits on newer compiler versions (useful
when performing a "git bisect"). See "lyxbuild --help" for more
information.

The previous home of lyx-tester was:

  https://gitlab.com/scottkosty/lyx-tester

development/lyx-tester/CreateQt5

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# Copyright 2014, Kornel Benko
+# Copyright 2014, Scott Kostyshak
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eu
+
+USERNAME=$( grep "/bin/bash" /etc/passwd | perl -pe 's/.*\/(\w+):\/bin\/bash/$1/g' | grep -v "root" )
+sudo -u "${USERNAME}" echo "If no error, then USERNAME is valid" >/dev/null || { echo "could not get the user name"; exit 1 ;}
+
+# Not used yet. Eventually add options.
+QTSTABLE=1
+QTDEV=1
+# TODO check that at least one of the above is set.
+# The git repo for Qt WebEngine is about 3.3G and LyX doesn't need it. If set
+# to 0, no need to download it or build it.
+QTWEB=0
+CLANG=1
+
+if [ "${CLANG}" = "1" ]; then
+  platform="linux-clang"
+else
+  platform="linux-g++"
+fi
+
+# This installs most dependencies.
+sudo apt-get -y build-dep qt5-default
+
+# needed (even if using GCC as main compiler) for QDoc. Otherwise, configure gives:
+#   "WARNING: QDoc will not be compiled, probably because libclang could not be located. This
+#   means that you cannot build the Qt documentation."
+# just need to install libclang-dev and the package containing the command llvm-config
+sudo apt-get -y install llvm libclang-dev
+
+# don't need to install if CLANG=0, but might as well (this way can test both
+# gcc and clang after installation).
+sudo apt-get -y install clang
+
+# needed for building Qt
+# Alternative: try to give the path directly to Qt when configuring or building.
+#              Look at environment variables PYTHONPATH and PYTHONNAME.
+ln -s /usr/bin/python3 /usr/bin/python
+
+# this turns on Qt's "GStreamer 1.0" to "yes" in configure output.
+sudo apt-get -y install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev
+
+# TODO:
+# TODO: also check if these should be here or in lyx-tester. any change in configure output???
+# If the following package is installed, xvkbd passes capital letters as lowercase
+# to LyX. This results in several keytest failures (such as hello-world).
+sudo apt-get -y purge libxcb-xkb-dev
+sudo apt-get -y install libxkbcommon-dev libxkbcommon-x11-dev      # This does not help xvkbd :(
+
+if [ "${QTWEB}" = "1" ]; then
+  # for an updated list of packages, see section for Qt WebEngine here:
+  #   https://wiki.qt.io/Building_Qt_5_from_Git
+  # see also here:
+  #   https://doc.qt.io/qt-5/qtwebengine-platform-notes.html
+  sudo apt-get -y install gperf bison flex libgl1-mesa-dev libnspr4-dev mesa-common-dev x11proto-scrnsaver-dev libcap-dev libnspr4-dev libnss3-dev libpci-dev libxss-dev x11proto-scrnsaver-dev
+  # configure requires that Python 2 must be installed for building Qt WebEngine
+  sudo apt-get -y install python2
+fi
+
+# the dir in which we download the source files
+qtsrc_d='/usr/src/qt'
+
+# This check is for convenience. On a virtual box, it is nice to
+# do a git clone and init once and then test variations through
+# cloning of virtual boxes.
+if [ ! -d "${qtsrc_d}" ]; then
+  sudo mkdir -p "${qtsrc_d}"
+  sudo chown ${USERNAME} "${qtsrc_d}"
+  cd "${qtsrc_d}"
+  sudo -u "${USERNAME}" git clone git://code.qt.io/qt/qt5.git qt5
+  cd qt5
+  #
+  # Qt's git branch policies:
+  #   http://qt-project.org/wiki/Branch-Guidelines
+  # Here is where we would check out a branch if we don't want dev. e.g.,
+  # check out a release:
+  #   sudo -u "${USERNAME}" git checkout v5.14.1
+  # checkout a branch (e.g., unreleased stable branch):
+  #   sudo -u "${USERNAME}" git checkout 5.14.2
+  # checkout a beta release:
+  #   sudo -u "${USERNAME}" git checkout "v5.15.0-beta2"
+  sudo -u "${USERNAME}" git checkout "v5.15.0"
+  #
+  # This is what downloads the bulk (the "git clone" above only downloads 15 MB).
+  if [ "${QTWEB}" = "1" ]; then
+    sudo -u "${USERNAME}" ./init-repository
+  else
+    # configure will automatically not try to build it if it is not init'd.
+    sudo -u "${USERNAME}" ./init-repository --module-subset=default,-qtwebengine
+  fi
+fi
+
+
+# create build dir and compile
+sudo mkdir /usr/BUILD
+sudo chown ${USERNAME} /usr/BUILD
+sudo -u ${USERNAME} mkdir /usr/BUILD/BuildQt5-dev
+cd /usr/BUILD/BuildQt5-dev
+# to get list of features, do the following (takes a minute to generate them):
+# configure -list-features
+# also here:
+# https://doc.qt.io/qt-5/configure-options.html
+#
+# -v adds verbose output (specifically helpful when there's an error)
+# TODO: switch back to -developer-build so I can help report breaks?
+sudo -u "${USERNAME}" "${qtsrc_d}/qt5/configure" -platform "${platform}" -opensource -nomake examples -nomake tests -confirm-license -v
+numCores=$( cat /proc/cpuinfo | grep processor | wc -l )
+sudo -u "${USERNAME}" make -j${numCores}
+
+# Uncomment this block if you want the "git" Qt to be used by the whole system.
+## Inform the OS about the new libraries
+#echo "/usr/BUILD/BuildQt5-dev/qtbase/lib" > qt5.conf
+#sudo cp qt5.conf /etc/ld.so.conf.d/qt5.conf
+#sudo ldconfig
+
+# Add qt-bin to own path
+# vi ~/.pam_environment
+#       Append '/usr/BUILD/BuildQt5-dev/qtbase/bin' to PATH
+#       e.g. 'PATH DEFAULT=${PATH}:/usr/local/lyx2.2/bin:/usr/local/texlive/2013/bin/x86_64-linux:/usr/BUILD/BuildQt5-dev/qtbase/bin'
+
+# Before compiling lyx, make sure that PATH contains /usr/BUILD/BuildQt5-dev/qtbase/bin

development/lyx-tester/DEBUGTIPS

@@ -0,0 +1,7 @@
+These are good ideas when debugging lyx-tester:
+
+run with no parallel
+run with TeX Live .iso
+do not do an apt-get update or dist-upgrade
+try to make it so an internet connection is not needed
+Either run all from ssh or run all manually (remember the bug of not exporting DISPLAY)

development/lyx-tester/README.md

@@ -0,0 +1,43 @@
+# install-tl-ubuntu
+
+## Usage
+
+```s
+sudo ./lyx-tester
+```
+
+## Description
+
+These scripts help with testing LyX. The main idea is that
+you can clone this repo into a fresh Ubuntu install (e.g. in a
+virtual box) and run all LyX tests with just one command. This
+script does the following:
+
+- git clones and compiles the LyX master branch
+- installs the dependencies needed to run the tests (e.g. R, knitr,
+LilyPond, DocBook)
+- installs language dependencies (Hebrew and Chinese fonts)
+- runs install-tl-ubuntu (see below)
+- runs all of the tests (tex2lyx, autotests, and export tests)
+
+lyx-tester runs [install-tl-ubuntu](https://github.com/scottkosty/install-tl-ubuntu), which does the following:
+
+- a full installation of the latest release of TeX Live.
+- notifies apt so that apt does not try to install the Ubuntu
+texlive-* packages as dependencies (e.g. if you do 'sudo apt-get
+install lyx')
+- installs (optionally) all of the LaTeX files that LyX templates and
+examples depend on
+- links to the folder where Ubuntu installs TeX files so that when you
+install packages with LaTeX (e.g. FoilTeX and noweb), they will be
+available.
+
+Finally, the script ssh-lyx-tester dispatches lyx-tester to an
+ssh-server. I use this with Amazon's EC2. All one has to do is run the
+following
+  ./ssh-lyx-tester <key-location> <server>
+
+## Authors
+
+Scott Kostyshak
+Kornel Benko

development/lyx-tester/TODO

@@ -0,0 +1,25 @@
+lyx-tester:
+- Test again on Amazon EC2. Here is some old commented out code:
+# The Amazon EC2 basic (400 MB of RAM) fails at linking stage if this flag is
+# on when building and LyX will run faster without debug symbols anyway.
+# Alternative:
+#     You could leave this alone and change -DLYX_RELEASE to 'ON'. AWS will
+#     also be able to build in this case. However, I think it's better to have
+#     ASSERTs turned on.
+#sed -i 's/-DLYX_DEBUG=ON/-DLYX_DEBUG=OFF/' lyxbuild >&2 \
+#    || { echo "ERROR: could not replace with sed" >&2; mainRet=1 ;}
+
+- Options:
+  argument for autotests
+  logdir into an option
+  runTests (lyx-tester might want to be used just to install
+        all prereqs and prep for tests or a complete system)
+  numCores
+
+ssh-lyx-tester:
+# We might want to transfer a key in order to test feature branches
+# scp -i "${key}" id_rsa ubuntu@${server}:/home/ubuntu/.ssh/ >/dev/null 2>&1 \
+#     || { echo "ERROR: could not scp the ssh key over." >&2; exit 1; }
+
+mylyxcd:
+# sanity checks: only 1 arg.

development/lyx-tester/convenience-functions.sh

@@ -0,0 +1,12 @@
+# This will match strings with Qt accelerators, like "so&me message",
+# without having to specify the ampersand.
+function git-grep-accel-robust ()
+{
+  # TODO preserve "git" args? i.e., allow user to not use -i, etc.
+  #
+  # Insert the string "&\?" after each character to allow for an ampersand
+  # to be anywhere.
+  # (https://unix.stackexchange.com/a/5981/197212)
+  search_str="$( echo "$1" | sed 's/.\{1\}/&\&\\\?/g')"
+  git grep -i "${search_str}"
+}

development/lyx-tester/ctest-reg-if-fail

@@ -0,0 +1,76 @@
+function add_suffix () {
+  base_s="$1"
+  if [ -e "${base_s}" ]; then
+
+    # we could rename the existing but let's just leave things as is.
+    # echo "${base_s} already exists."
+    #if [ -e "${base_s}_1" ]; then
+    #  echo "ERROR: ${base_s}_1 already exists. Unexpected." >&2
+    #else
+    #  # we are renaming the *existing* dir.
+    #  echo "Suffixing '_1' to existing dir name ({base_s})."
+    #  sudo mv "${base_s}" "${base_s}_1"
+    #fi
+
+    # The non-suffixed dir should be interpreted as number 1.
+    suffix=2
+    while [ -e "${base_s}_${suffix}" ]; do
+      suffix=$((suffix + 1))
+    done
+    base_s="${base_s}_${suffix}"
+  fi
+  echo "${base_s}"
+}
+
+# "Missing glyphs" from Noto Tibetan:
+# supported-languages_polyglossia_pdf4_systemF
+#
+# %28APA%29,_v._6: not sure why these are failing (the apa6.cls exists) but not worth the time since obsolete.
+#
+# supported-languages_polyglossia_.*_systemF: The LuaTeX changes started failing at 99299169. I think XeTeX failed before. I need to figure out the Noto fonts thing.
+#
+# Europe_CV: see here: https://github.com/gsilano/EuropeCV/issues/29
+# There is a patch but is not backwards-compatible so it's not clear if
+# it will be merged.
+#
+# BATCH_compare-test: I'm not sure why this fails, but no time to look into it.
+#
+# BATCH_vcs-info, BATCH_outline-beamer: these tests fail when configure.ac has, e.g., "beta".
+#   We know what we need to do to fix this, but just need to do it.
+#   sk: see personal email here for more info:
+#     Message-ID:<20221221165514.pbfrlpt77o3my55o@gary>
+#
+# Astronomical: the required file aastex62.cls is not found anymore. Probably there is a new version with a new file name.
+#               TODO: look into ^this when I have time.
+#               Jean-Pierre helped transition things to a new class (and file name) version.
+#
+ctest_exclude='(supported-languages_polyglossia_.*_systemF|(%28APA%29,_v._6|Astronomical).*(pdf|dvi|lyx22|lyx23)|BATCH_compare-test|BATCH_outline-beamer|BATCH_vcs-info)'
+# timeout of 10m.
+function ctest-reg-if-fail ()
+{
+  ctest -E "${ctest_exclude}" --timeout 600 $@
+  ret="$?"
+
+  failed_log="Testing/Temporary/LastTestsFailed.log"
+  # there is no file if no failures.
+  if [ -e "${failed_log}" ]; then
+    if [ -e "/tmp/LastTestsFailed" ]; then
+      target_f="$( add_suffix "/tmp/LastTestsFailed" )"
+      # we want the root one to be the newest.
+      mv /tmp/LastTestsFailed "${target_f}"
+    fi
+    cp -f "${failed_log}" /tmp/LastTestsFailed
+  fi
+
+  return "${ret}"
+}
+# This tests all of the ones excluded above. Good to check once in a while to
+# see if we can "uninvert" them (in our 'ctest_exclude' variable that is).
+alias ctest-reg-if-fail-INVERTED='ctest -R "${ctest_exclude}"'
+# TODO: this doesn't work. Ask on SO? Use my specific case (of ctest)
+source /usr/share/bash-completion/completions/ctest
+complete -F _ctest ctest-reg-if-fail
+#
+# useful when want to test mainly for polyglossia regressions
+alias ctest-reg-if-fail-polyglossia='ctest -E "${ctest_exclude}" -R "systemF" --timeout 600'
+

development/lyx-tester/lyx-tester

@@ -0,0 +1,783 @@
+#!/usr/bin/env bash
+
+
+# File: lyx-tester
+
+# Usage:
+# sudo ./lyx-tester [OPTIONS]
+
+# Description:
+# This script does the following:
+#   - makes changes to /etc/apt/sources.list
+#   - apt-get update
+#   - runs install-tl-ubuntu to set up a full TeX Live installation
+#   - installs dependencies of LyX's export tests
+#   - builds master branch of LyX (jobs=#cores)
+#   - runs the tests (tex2lyx, export, and autotests)
+
+# Progress and profiling are logged to STDOUT.
+# Important errors are logged to STDERR.
+# Annoying STDOUT and STDERR messages are redirected to a file descriptor
+# (which is connected to a file *EXTRAS.log by default) in case they are
+# actually useful for debugging.
+
+# If the directory "ChineseFonts" exists in the current directory,
+# lyx-tester will install system fonts located in ChineseFonts/xetex and
+# TrueType fonts located in ChineseFonts/pdftex.
+# For more details, see 'install-tl-ubuntu --help'.
+# Note that we do not currently test Chinese documents with pdfTeX.
+
+
+# Copyright 2013-2015, Scott Kostyshak
+# Copyright 2013-2015, Kornel Benko
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+set -o nounset
+
+ubuntu_ver="$( lsb_release -sr )"
+ubuntu_ver_num="${ubuntu_ver/\./}"
+
+function checkTerm() {
+  # check if the column is 0 and if not start a new line (useful bc of the progress bar).
+  # based on code from: http://stackoverflow.com/questions/2575037/how-to-get-the-cursor-position-in-bash
+  exec < /dev/tty
+  oldstty=$(stty -g)
+  stty raw -echo min 0
+  # on my system, the following line can be replaced by the line below it
+  echo -en "\033[6n" > /dev/tty
+  # tput u7 > /dev/tty    # when TERM=xterm (and relatives)
+  IFS=';' read -r -d R -a pos
+  stty $oldstty
+  # change from one-based to zero based so they work with: tput cup $row $col
+  row=$((${pos[0]:2} - 1))    # strip off the esc-[
+  col=$((${pos[1]} - 1))
+  if [ "${col}" = "0" ]; then
+    exit
+  else
+    echo "\n"
+  fi
+}
+
+function ECHOPREFIX() {
+  if [[ "$-" =~ i ]]; then
+    exit
+  else
+    # At some point (first noticed on Ubuntu 22.04), the checkTerm magic
+    # caused some problems so disable for now.
+    # TODO: check back on this and see if it's worth debugging and putting back in.
+    #echo -e "$(checkTerm)$( basename "$0" ): "
+    echo -e "$( basename "$0" ): "
+  fi
+}
+
+function ERRORPREFIX() {
+  echo "$(ECHOPREFIX)ERROR: "
+}
+
+function WARNINGPREFIX() {
+  echo "$(ECHOPREFIX)WARNING: "
+}
+
+function show_help() {
+  echo -e "
+  USAGE
+  sudo ./lyx-tester [OPTIONS]
+
+  DESCRIPTION
+  A script to test various components of LyX
+
+  This script can be run on a fresh installation of Ubuntu and will git clone and compile the
+  latest development version of LyX and run the extensive suite of tests in LyX.
+
+  OPTIONS
+  -b <branch>, --itl-branch <branch>     \e[00;36m Checkout <branch> of install-tl-ubuntu.                          \e[00m
+  -d, --only-install-deps                \e[00;36m Only install some (not all) dependencies, mostly using apt-get.  \e[00m
+  -h, --help                             \e[00;36m Show this help and exit.                                         \e[00m
+  -n, --no-test                          \e[00;36m Do not run the ctests. This option is useful if you just want to
+                                            quickly get a development environment for LyX.                          \e[00m
+  -q <source>, --qt-from <source>        \e[00;36m How Qt should be installed. <source> can be 'ubuntu' or 'git'.
+                                            'git' compiles the development version of Qt from the cloned Git
+                                            repository. This requires a lot of space: if the 'git' option
+                                            is chosen for a 'lyx-tester' run on a fresh install of Ubuntu, a total
+                                            of about 37G is needed.                                                 \e[00m
+  -u, --no-workarounds                   \e[00;36m Do not apply workarounds for TeX issues. e.g., CTAN packages where
+                                            a fix/workaround is known but a new version has not been released.      \e[00m
+  -x <options>, --itl-options <options>  \e[00;36m <options> will be appended to the default options that
+                                            lyx-tester calls install-tl-ubuntu with. For a list of options,
+                                            see 'install-tl-ubuntu --help'.                                         \e[00m
+
+  AUTHOR
+  Scott Kostyshak
+  Kornel Benko
+"
+}
+
+ARGS=$( getopt -o "x:q:b:nhd" -l "itl-options:,qt-from:,itl-branch:,no-test,help,only-install-deps" \
+      -n "$( basename "$0" )" -- "$@" )
+# If bad arguments, exit (getopt will have already written the error to STDERR).
+[ "$?" = "0" ] || { exit 1; }
+
+eval set -- "$ARGS"
+
+ITL_BRANCH=''
+ITL_OPTIONS=''
+# 'git' means use the development branch of Qt 5 (requires a lot of space and time to compile)
+QT_FROM='ubuntu'
+RUN_TESTS=1
+WORKAROUNDS=1
+ONLY_DEPS=0
+#
+# todo: Is it worth fixing the autotests for Wayland?
+# currently no option to change this. Make an option
+# if we keep the framework around.
+autotests=1
+
+
+while true
+do
+  case "$1" in
+  -b|--itl-branch)
+      ITL_BRANCH="$2"
+      shift 2
+      ;;
+  -d|--only-install-deps)
+      ONLY_DEPS=1
+      shift 1
+      ;;
+  -h|--help)
+      show_help
+      exit 0
+      ;;
+  -n|--no-test)
+      RUN_TESTS=0
+      shift 1
+      ;;
+  -q|--qt-from)
+      if [[ "$2" = "ubuntu" || "$2" = "git" ]]; then
+        QT_FROM="$2"
+        shift 2
+      else
+        echo "$(ERRORPREFIX) --qt-from supports 'ubuntu' and 'git'"
+        exit 1
+      fi
+      ;;
+  -u|--no-workarounds)
+      WORKAROUNDS=0
+      shift 1
+      ;;
+  -x|--itl-options)
+      ITL_OPTIONS="$2"
+      shift 2
+      ;;
+  --)
+      shift 1
+      break;;
+  *)
+      echo -e "${ERRORPREFIX}the getopt command must be out of sync. Please report this."\\n\
+          "The following flag is not recognized: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+# I do this because I run this script with sudo, but I want some files to be
+# created by the user. Note that other ways of getting the user name (such as
+# logname, whoami, and 'who am i') seem to fail in some cases.
+users="$(grep "/bin/bash" /etc/passwd \
+     | perl -pe 's/.*\/(\w+):\/bin\/bash/$1/g' | grep -v "root")"
+nusers="$(echo "${users}" | wc -l)"
+if [ "${nusers}" = 1 ]; then
+  USERNAME="${users}"
+else
+  USERNAME="$(logname)"
+fi
+sudo -u "${USERNAME}" echo "If no error, then USERNAME is valid" >& /dev/null \
+  || { echo "$(ERRORPREFIX)could not get the user name" >&2; exit 1; }
+
+[ "$(whoami)" = "root" ] \
+  || { echo "$(ERRORPREFIX)this script must be run with 'sudo'." >&2; exit 1; }
+
+mainDir="$(pwd)"
+LOGDIR="${mainDir}/lyx-tester-logs"
+if [ ! -d "${LOGDIR}" ]; then
+  sudo -u "${USERNAME}" mkdir "${LOGDIR}"
+fi
+
+exec > >(tee -a "${LOGDIR}/$( basename "$0" ).log")
+exec 2> >(tee -a "${LOGDIR}/$( basename "$0" ).log")
+exec 3>> "${LOGDIR}/lyx-tester_EXTRAS.log"
+
+date
+
+# For some errors, we exit. For others (non-fatal) we continue but at the end
+# report that something went wrong (i.e. mainRet != 0).
+mainRet=0
+
+# Main advantage is that it downloads/installs TeX Live at same time as
+# cloning/building LyX.
+parallel=1
+
+numCores=$( cat /proc/cpuinfo | grep processor | wc -l )
+
+# Number of seconds until each ctest times out.
+# The longest one is MergedManuals_xhtml and took 3935 seconds on AWS basic.
+CTEST_TIMEOUT=4000
+
+# enable Multiverse to install foiltex for testing examples/Foils.lyx.
+# it's possible this is already enabled after a fresh install,
+# (if for Ubuntu install we clicked the third-party checkbox I think)
+perl -pi -e 'next if /-backports/; s/^# (deb [^ ]+ [^ -]+ multiverse)$/$1/' /etc/apt/sources.list \
+    || { echo "$(ERRORPREFIX)perl failed to add multiverse" >&2; exit 1; }
+#
+# The following is for build-dep lyx and build-dep r-cran-digest
+#
+# This chunk is shared by lyx-tester/lyx-tester, lyx-tester/vbox_prep.sh, master.sgk, vboxBigInstall.sh.
+# Needed for "apt-get build-dep".
+if [ ${ubuntu_ver_num} -ge 2404 ]; then
+  # Don't really need to sub the entry for security, but easy to just keep as is.
+  # use '$' so that this command can be run multiple times and will only make the sub once.
+  sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources || { echo "$(ERRORPREFIX) failed to add source for universe (needed for build-dep)" >&2; exit 1; }
+else
+  sudo perl -pi -e 'next if /-backports/; s/^# (deb-src [^ ]+ [^ -]+ universe)$/$1/' /etc/apt/sources.list || { echo "$(ERRORPREFIX)perl failed to add source for universe (needed for build-dep)" >&2; exit 1; }
+fi
+sudo apt-get --yes update > apt-get-update.log 2>&1 || { echo "$(WARNINGPREFIX)apt-get update failed after universe src and multiverse adds." >&2; }
+
+
+echo "$(ECHOPREFIX)installing pre-reqs for install-tl-ubuntu..."
+apt-get --yes install git >&3 2>&1
+
+if [ -z "${ITL_BRANCH}" ]; then
+  maybeBranch=""
+else
+  maybeBranch="--branch ${ITL_BRANCH}"
+fi
+sudo -u ${USERNAME} git clone ${maybeBranch} "https://github.com/scottkosty/install-tl-ubuntu.git" install-tl-ubuntu.git >&3 2>&1 \
+  || { echo "$(ERRORPREFIX)could not git clone install-tl-ubuntu" >&2; exit 1; }
+
+(
+  if dpkg -s texlive-local > /dev/null 2>&1; then
+    echo "$(ECHOPREFIX)texlive-local already installed, skipping equivs."
+  else
+    cd ./install-tl-ubuntu.git &&
+      ./install-tl-ubuntu --only-apt-and-dpkg --hebrew --log-dir "${LOGDIR}" ${ITL_OPTIONS} ||
+      { echo "$(ERRORPREFIX)install-tl-ubuntu --only-apt-and-dpkg failed." >&2; exit 1; }
+  fi
+)
+
+function install_main_deps() {
+  if [ "${autotests}" = "1" ]; then
+    # xvkbd is not needed because LyX includes it but I would like to test
+    # the system install so that we can one day remove it from LyX sources.
+    # The Italian packages are needed for Tomasso's tests.
+    autotestDeps="libxaw7-dev libxtst-dev pcregrep wmctrl xvkbd \
+                 firefox-locale-it language-pack-it language-pack-it-base"
+  else
+    autotestDeps=''
+  fi
+
+  #     r-base for Sweave/knitr
+  #     ghostcript for ps2pdf
+  #     foiltex for the examples/Foils.lyx testing (we enabled Multiverse for this)
+  #     gnumeric (for an example I think and maybe also for tex2lyx (40 MB)
+  #     noweb for Literate.lyx and noweb2lyx.lyx
+  #     the following fonts are helpful for Japanese with ps2pdf:
+  #     fonts-ipafont-mincho fonts-takao-mincho fonts-ipafont-nonfree-jisx0208 fonts-ipaexfont-mincho
+  #     the following fonts are used for exporting Farsi docs with XeTeX
+  #     fonts-freefarsi
+  #     perl-tk is for 'tlmgr --gui'
+  #     clang is now used by default by lyxbuild
+  #     libc++abi-dev is for testing compilation with -stdlib=libc++
+  #     libtry-tiny-perl is for Kornel's URL tests (which are not run by default)
+  #     doxygen is for "make doxydoc"
+  #     the following fonts are for exporting zh_CN docs with XeTeX system fonts:
+  #     fonts-wqy-microhei
+  #     the following fonts are for Korean splash:
+  #     fonts-nanum
+  #     python3-polib is needed for updating layouttranslations ("make layouttranslations1")
+  #     the following are for future tests of Arabic docs. See LyX hash 36b9b973.
+  #     note that it is necessary to remove the woff files which is done below
+  #     fonts-sil-scheherazade
+  #     libxml2-utils for xmllint for the xhtml tests
+  #     libfontconfig1-dev is needed for newest Qt, so with -fontconfig switch fonts work
+  #     python3-pygments is for minted
+  #     the following are for compilation of supported-languages.lyx.
+  #     fonts-noto-core is a large (30 MB) package, needed for Ethiopic and Devanagari.
+  #     comparepdf is searched for by 'cmake' for ams-import tests
+  #     libfile-slurp-perl is needed for 5 tests: BATCH_outline-beamer, \
+  #         BATCH_vcs-info, BATCH_AMS-import, BATCH_SAVE-as, BATCH_compare-test
+  #     The following are for building with Qt6:
+  #         qt6-base-dev libqt6opengl6-dev qmake6 qmake6-bin libqt6opengl6-dev qmake6 qmake6-bin qt6-base-dev qt6-svg-dev qt6-wayland
+
+
+  #     The following are useful if we decide to not rely on TL fonts for system fonts (used for, e.g., XeTeX).
+  #
+  #     Neeeded for: DEFAULTOUTPUT_export/export/latex/languages/supported-languages_polyglossia-XeTeX_pdf4_systemF
+  #       fonts-ipafont fonts-hosny-amiri fonts-baekmuk
+  #     couldn't find a simple Ubuntu package replacement for: Monomakh
+  #     or can copy just this one:
+  #       /usr/share/texlive/texmf-dist/fonts/opentype/public/fonts-churchslavonic/MonomakhUnicode.otf
+  #     it's here: texlive-fonts-extra, but that is a large dependency (the .deb is 509 MB).
+  #
+  # Without TL font link, this fixes compilation for: export/export/latex/CJK/en-de-el-ru-ko_utf8_pdf4_systemF
+  # 'fonts-unfonts-extra' provides these fonts:
+  #     UnJamoBatang
+  #     UnJamoDotum
+  #     UnTaza
+  #
+  # Without TL font link, this fixes compilation for many ja documents:
+  # fonts-ipaexfont
+
+
+  apt-get -y install --no-install-recommends r-base ghostscript imagemagick \
+    foiltex gnumeric noweb ${autotestDeps} \
+    fonts-ipafont-mincho fonts-takao-mincho fonts-ipafont-nonfree-jisx0208 fonts-ipaexfont-mincho \
+    fonts-freefarsi \
+    perl-tk \
+    clang \
+    libc++abi-dev \
+    libtry-tiny-perl \
+    doxygen \
+    fonts-wqy-microhei \
+    fonts-nanum \
+    fonts-sil-scheherazade \
+    libxml2-utils \
+    libfontconfig1-dev \
+    python3-polib \
+    python3-pygments \
+    python3-pygments \
+    gnuplot-nox \
+    fonts-noto-core \
+    comparepdf \
+    libfile-slurp-perl \
+    qt6-base-dev libqt6opengl6-dev qmake6 qmake6-bin libqt6opengl6-dev qmake6 qmake6-bin qt6-base-dev qt6-svg-dev qt6-wayland \
+    fonts-ipafont fonts-hosny-amiri fonts-baekmuk fonts-unfonts-extra fonts-ipaexfont \
+    >&3 2>&3 \
+    || { echo "$(ERRORPREFIX)"install of r-base or a LyX test dependency failed. \
+         PPA might be updating and causing dependency version mismatches. >&2; exit 1; }
+
+  echo "$(ECHOPREFIX)Running 'build-dep r-cran-digest'..."
+  apt-get -y build-dep r-cran-digest >&3 2>&1 \
+    || echo "$(WARNINGPREFIX)"build-dep r-cran-digest failed >&2
+  echo "$(ECHOPREFIX)Running 'build-dep r-cran-digest'... done."
+
+  echo "$(ECHOPREFIX)installing LyX pre-reqs..."
+  qtlibs=''
+  if [ ${QT_FROM} = "ubuntu" ]; then
+    # if building Qt from source, we do not want these to be used accidentally.
+    qtlibs='libqt5svg5-dev'
+  fi
+  # One could alternatively use autotools instead of CMake.
+  # We don't need 'hunspell' (1MB) but it is included to test building with support for it.
+  # If you take it out, edit the CMake build flags.
+  # libqt5x11extras5-dev is needed for CMake to detect that X11 is used when using Qt 5 (needed for keytests).
+  # librsvg2-bin is for converting SVG to PNG and PDF (needed since we use svg in manuals). Could use inkscape instead. See configure.py for more details. This sets up the e.g. SVG -> PDF (graphics) converter (SVG -> pdf6).
+  apt-get --yes build-dep --no-install-recommends lyx >&3 2>&1 \
+    || echo "$(WARNINGPREFIX)build-dep lyx failed. Source package is probably not available." >&2
+  apt-get --yes install --no-install-recommends cmake hunspell libhunspell-dev libqt5x11extras5-dev librsvg2-bin ${qtlibs} >&3 2>&1 \
+    && echo "$(ECHOPREFIX)installing LyX pre-reqs successful." \
+    || echo "$(WARNINGPREFIX)LyX pre-reqs failed." >&2
+
+}
+# Placement of this function call is after install-tl-ubuntu equivs because we don't want Ubuntu bringing
+# in any texlive packages that these depend on.
+#
+# Install LyX auxiliaries before LyX's configure so that it will find them.
+#
+# This chunk also goes before install-tl-ubuntu so that when we create
+# a symlink to Ubuntu's TeX dir in install-tl-ubuntu, it's not dangling
+# (e.g. chmod will not give an error). Not strictly necessary though.
+install_main_deps
+if [ "${ONLY_DEPS}" = "1" ]; then
+  exit 0
+fi
+
+# remove the woff files from installation of package fonts-sil-scheherazade
+# since dvipdfmx does not support them:
+# http://tex.stackexchange.com/questions/330195/how-to-set-up-the-font-scheherazade-for-use-with-xelatex
+rm -rf /usr/share/fonts/woff
+
+# set the policies for ImageMagick so that it allows more conversions. This is
+# necessary for many of the tests because included graphics need to be
+# converted to various formats depending on the engine that is used (e.g.,
+# latex, pdflatex).
+sed -i 's/rights="none" pattern="PS"/rights="read|write" pattern="PS"/' /etc/ImageMagick-6/policy.xml &&
+sed -i 's/rights="none" pattern="EPS"/rights="read|write" pattern="EPS"/' /etc/ImageMagick-6/policy.xml &&
+sed -i 's/rights="none" pattern="PDF"/rights="read|write" pattern="PDF"/' /etc/ImageMagick-6/policy.xml
+
+# Create personal library. That way, don't need "sudo" to install packages.
+# (https://stackoverflow.com/a/43283085/1376404)
+# On Ubuntu 19.10, this creates the following directory:
+#   ~/R/x86_64-pc-linux-gnu-library/3.6
+# alternative:
+#R -e 'dir.create(path = Sys.getenv("R_LIBS_USER"), showWarnings = FALSE, recursive = TRUE)'
+R_maj_dot_min="$(apt-cache policy r-base | grep -oP '(?<=Candidate: )\d\.\d')"
+sudo -u ${USERNAME} mkdir -p "/home/${USERNAME}/R/x86_64-pc-linux-gnu-library/${R_maj_dot_min}"
+# warn=2 turns warnings into errors (this causes a non-zero exit if there are
+# failed installs of packages)
+sudo -u ${USERNAME} R -e "options(warn=2); options(repos= 'http://lib.stat.cmu.edu/R/CRAN/'); install.packages('knitr')" >&3 2>&3 \
+  || { echo "$(WARNINGPREFIX)knitr install failed." >&2; }
+
+if [ -d "ChineseFonts" ]; then
+  echo "$(ECHOPREFIX)installing Chinese Fonts."
+  ZH='--usr-fonts ChineseFonts/xetex --truetype-dir ChineseFonts/pdftex'
+else
+  echo "$(ECHOPREFIX)not installing Chinese Fonts."
+  ZH=''
+fi
+
+echo "$(ECHOPREFIX)forking install-tl-ubuntu."
+( cd ./install-tl-ubuntu.git && ./install-tl-ubuntu --more-tex ${ZH} --hebrew --log-dir "${LOGDIR}" --no-apt-or-dpkg ${ITL_OPTIONS} ) &
+pid_tl_ubuntu=$!
+echo "$(ECHOPREFIX)pid_tl_ubuntu is ${pid_tl_ubuntu}"
+
+TLRET=0
+if [ "$parallel" != "1" ]; then
+  echo "$(ECHOPREFIX)waiting for install-tl-ubuntu to finish (not in parallel)..."
+  wait ${pid_tl_ubuntu} &&
+    echo "$(ECHOPREFIX)$(date +%H:%M): install-tl-ubuntu successful." \
+    || { echo "$(ERRORPREFIX)$(date +%H:%M): install-tl-ubuntu failed." >&2; TLRET=1; mainRet=1; }
+fi
+
+
+sudo -u ${USERNAME} mkdir -p "/home/${USERNAME}/lyxbuilds/master" || { echo "$(ERRORPREFIX)mkdir failed." >&2; exit 1; }
+sudo -u ${USERNAME} mkdir -p "/home/${USERNAME}/lyxbuilds/master-gcc" || { echo "$(ERRORPREFIX)mkdir failed." >&2; exit 1; }
+echo "$(ECHOPREFIX)starting child process git clone and then lyxbuild..."
+LYX_DIR="/home/${USERNAME}/lyxbuilds/master/repo"
+LYX_DIR_GCC="/home/${USERNAME}/lyxbuilds/master-gcc/repo"
+# TODO make these two options of lyx-tester
+LYXDEV=0 # set to 1 if you want to fetch features with write access and have the ssh key to do so.
+if [ "${QT_FROM}" = "git" ]; then
+  echo "$(ECHOPREFIX)building Qt 5 stable branch from git..."
+  ./CreateQt5 > "${LOGDIR}/CreateQt5.log" 2>&1
+  if [ "$?" = "0" ]; then
+    maybeQtDirOption='--qt-build-dir="/usr/BUILD/BuildQt5-dev"'
+  else
+    echo "$(WARNINGPREFIX)could not build Qt 5 (will try to proceed with apt packages instead)." >&2
+    maybeQtDirOption=''
+    apt-get --yes install --no-install-recommends "${qtlibs}"
+  fi
+else
+  maybeQtDirOption=''
+fi
+
+# TODO: For some reason, a git clone of LyX often fails. I thus took it out of the
+# subprocess. Perhaps that will help? Perhaps an environment variables issue?
+# update: taking it out of the subprocess did not help. Put it back in if there's an
+# advantage to it.
+#
+# similar code in install-tl-ubuntu
+timesToTry=5
+timesTried=0
+fail=0
+success=0
+while (( timesTried != timesToTry )); do
+    timesTried=$(( timesTried + 1 ))
+
+    # For some reason this fails in vbox when run in the script, but succeeds if I do the clone
+    # manually. I didn't figure out the root reason.
+    # Instead, now we clone from a mirror, then retarget the remote and re-pull.
+    # lyx_git_repo="git://git.lyx.org/lyx"
+    #
+    # todo: check if there is a resume feature to "git clone". When I last checked, there was
+    #       no such straight-forward feature.
+    #
+    # The mirror is also ~2x faster.
+    # TODO: use Yuriy's instead, which is more up to date:
+    # https://github.com/magistere/lyx2
+    lyx_git_repo="https://github.com/cburschka/lyx"
+
+    echo -e "$(ECHOPREFIX)attempting to clone LyX for time number ${timesTried}... using the following command:\n git clone \"${lyx_git_repo}\""
+    # temporary debugging of why git clone often fails.
+    # env > "/home/${USERNAME}/Desktop/env-before-git-clone-of-lyx.log"
+    # sudo -u "${USERNAME}" env > "/home/${USERNAME}/Desktop/env-USER-before-git-clone-of-lyx.log"
+    sudo -u "${USERNAME}" git clone "${lyx_git_repo}" "${LYX_DIR}"
+    if [ "$?" != "0" ]; then
+        fail=1
+        echo "$(WARNINGPREFIX)clone failed" >&2
+    fi
+    if [ "${fail}" = 0 ]; then
+        success=1
+        break
+    else
+        (( timesTried != timesToTry )) && echo "$(ECHOPREFIX)we will wait 30s, then try to restart the clone (from scratch)." >&2
+        rm -rf "${LYX_DIR}"
+        sleep 30s
+    fi
+done
+if [ "${success}" = "1" ]; then
+    echo "$(ECHOPREFIX)LyX was successfully cloned."
+else
+    echo "$(WARNINGPREFIX)git clone of LyX failed." >&2
+fi
+
+(
+  (
+    cd "${LYX_DIR}"
+    # Now we remove the clone remote and swap in the upstream remote
+    sudo -u ${USERNAME} git remote remove origin
+    sudo -u ${USERNAME} git remote add origin "git://git.lyx.org/lyx"
+    sudo -u ${USERNAME} git fetch --all >/dev/null 2>&1 || echo "$(ERRORPREFIX)'git fetch --all' failed." >&2
+    sudo -u ${USERNAME} git branch --set-upstream-to=origin/master master >/dev/null 2>&1 || echo "$(ERRORPREFIX)'git branch --set-upstream-to=origin/master master' failed." >&2
+    sudo -u ${USERNAME} git pull > /dev/null 2>&1 || echo "$(ERRORPREFIX)'git pull' failed." >&2
+
+    # TODO make an option
+    patches_d="/home/${USERNAME}/Downloads/lyxPatches"
+    if [ -d "${patches_d}" ]; then
+      echo "git am'ing .patch files found in ${patches_d}"
+      # only local config. Needed to git am patches.
+      sudo -u ${USERNAME} git config user.name "lyx-tester"
+      sudo -u ${USERNAME} git config user.email "lyx-tester"
+      sudo -u ${USERNAME} git am "${patches_d}"/*.patch
+    fi
+
+    if [ "${LYXDEV}" = "1" ]; then
+      # note that it needs to access the ssh keys of $USER
+      sudo -u "${USERNAME}" git remote add features git@git.lyx.org:features.git
+    else
+      sudo -u ${USERNAME} git remote add features git://git.lyx.org/features.git
+    fi
+    sudo -u "${USERNAME}" git fetch features >/dev/null 2>&1 || echo "$(ERRORPREFIX)'git fetch features' failed." >&2
+  )
+  sudo -u "${USERNAME}" cp -r "${LYX_DIR}" "${LYX_DIR_GCC}" &&
+  cd "${LYX_DIR_GCC}" &&
+  # need to source /etc/environment for qt build directory (in case using git Qt source)
+  sudo -u ${USERNAME} bash -c "source /etc/environment && "${mainDir}/lyxbuild" -j${numCores} --gcc ${maybeQtDirOption}" &&
+  echo "$(ECHOPREFIX)GCC build successful." \
+    || echo "$(WARNINGPREFIX)GCC build failed." >&2
+  cd "${LYX_DIR}" &&
+  sudo -u ${USERNAME} bash -c "source /etc/environment && "${mainDir}/lyxbuild" -j${numCores} ${maybeQtDirOption}" &&
+  echo "$(ECHOPREFIX)Clang build successful."
+) &
+# This is the PID of the subshell.
+pid_lyxdevel=$!
+echo "$(ECHOPREFIX)pid_lyxdevel is $pid_lyxdevel"
+
+echo "$(ECHOPREFIX)$(date +%H:%M): waiting on lyx-devel clang and gcc builds..."
+wait ${pid_lyxdevel} &&
+  echo "$(ECHOPREFIX)$(date +%H:%M): lyx-devel git clone and lyx cmake build successful." \
+  || { echo "$(ERRORPREFIX)lyx-devel gcc build failed." >&2; mainRet=1; }
+
+if [ "${parallel}" = "1" ]; then
+  echo "$(ECHOPREFIX)$(date +%H:%M): waiting on install-tl-ubuntu..."
+  wait ${pid_tl_ubuntu} &&
+    echo "$(ECHOPREFIX)$(date +%H:%M): install-tl-ubuntu successful." \
+    || { echo "$(ERRORPREFIX)install-tl-ubuntu failed." >&2; TLRET=1; mainRet=1; }
+fi
+#
+# when I use switch-tl-to, I do not want "sudo fc-cache -fsv" to be run each time.
+# however, a couple of tests are missing fonts from Ubuntu repos, so do not
+# run for now.
+# source /path/to/install-tl-ubuntu/switches && switch-sys-fonts-to-ubuntu
+
+# Reload path, otherwise kpsewhich and texhash will not be found.
+# Also, pdflatex will not be found when doing reconfigure for tests.
+source /etc/environment
+
+# This is needed for, e.g., hollywood.cls and lyxchess.sty.
+# alternative: install LyX with make install.
+if [ "${TLRET}" = 0 ]; then
+  cd "$( kpsewhich --var-value TEXMFLOCAL )/tex/" \
+    || { echo "$(WARNINGPREFIX)kpsewhich failed." >&2; mainRet=1; }
+  ln -s "${LYX_DIR}/lib/tex" LyXTeXDir \
+    || { echo "$(WARNINGPREFIX)linking LyX tex files failed." >&2; mainRet=1; }
+  texhash >&3 2>&1 || { echo "$(ERRORPREFIX)texhash failed." >&2; mainRet=1; }
+fi
+
+if [ "${WORKAROUNDS}" = "1" ]; then
+  echo "Performing workarounds..."
+
+  echo "Making fonts needed by some of our Russian documents..."
+  # This makes it so the /ru/ texF pdf5 and dvi3 tests all pass on first run.
+  # These mktextfm commands are triggered by other tests, e.g., ru/EmbeddedObjects_pdf2,
+  # so the affected tests pass on the second run; but we might as well get things running
+  # on the first ctest run.
+  #
+  # todo: there are some ideas here for how to automate finding the needed fonts.
+  # Currently, what I do is run the corresponding export in the GUI and then look at
+  # the log (it mentions the exact font name).
+  # https://tex.stackexchange.com/questions/649259/can-i-ask-luatex-to-run-mktextfm
+  #
+  fonts_arr=(
+    # I'm not sure if all of these are (still) necessary.
+    larm1000
+    larm1200
+    lass1200
+    lasx1200
+    lasx2488
+    larm1728
+    larm2488
+    labx2488
+    #
+    # The following became necessary I think with TeX Live 2023.
+    # Intro.lyx
+    larm2074
+    larm1440
+    larm0700
+    larm0800
+    # Math.lyx
+    larm0600
+    larm0500
+    larm1095
+    larm3583
+  )
+  for f in "${fonts_arr[@]}"; do
+    echo "Making font ${f}..."
+    sudo -u ${USERNAME} /opt/texbin/mktextfm "${f}" > /dev/null 2>&1 || { echo "$(ERRORPREFIX)mktextfm failed for ${f}." >&2; mainRet=1; }
+  done &&
+  echo "Done making fonts (successful)."
+
+  # This is not about a TeX Live package.
+  echo "  workaround for Ubuntu package 'noweb'..."
+  noweb_sty_f="/usr/share/texmf/tex/plain/misc/noweb.sty"
+  if [ -f "${noweb_sty_f}" ]; then
+    if file -i "${noweb_sty_f}" | grep -v 'utf-8' >/dev/null ; then
+      echo "Workaround is still needed for package 'noweb'."
+      # This fix is merged upstream but there has been no new release
+      # https://github.com/nrnrnr/noweb/pull/12
+      TEMP_F="$( mktemp )"
+      iconv -f latin1 -t utf8 "${noweb_sty_f}" -o "${TEMP_F}" &&
+      mv -f "${TEMP_F}" "${noweb_sty_f}"
+      # fix permissions so can be read by all
+      chmod +r "${noweb_sty_f}"
+    else
+      # Even when the fix makes it into Ubuntu, maybe still keep this code in
+      # in case we want to run lyx-tester on an older Ubuntu version?
+      echo "WARNING: workaround is no longer needed for package 'noweb'." >&2
+    fi
+  else
+    echo "Warning: noweb.sty not found. Skipping workaround." >&2
+  fi
+
+  echo "Done."
+fi
+
+# We install Lilypond here because we need to make symlinks first.
+# note that Lilypond needs kpsewhich
+# Starting with 19.04, without this I get an error "kpsewhich not found",
+# perhaps because of hardcoding of the path to kpsewhich?
+# TODO Report bug/enhancement upstream?
+#      Or move this workaround to install-tl-ubuntu?
+ln -s /opt/texbin/kpsewhich /usr/bin/kpsewhich
+apt-get --yes install --no-install-recommends lilypond >&3 2>&1 \
+    || { echo "$(ERRORPREFIX)lilypond failed to install." >&2; mainRet=1; }
+# I am not sure we need this symlink anymore, but we leave it just in case
+# For example, an update of the lilypond package might trigger the same script
+# that needs this hardcoding.
+# rm /usr/bin/kpsewhich
+
+# reconfigure LyX (it was reconfigured in lyxbuild, which if run in parallel
+# probably happened before the TeX Live installation finished).
+(
+  cd "${LYX_DIR}/../user-dir"
+  sudo -u ${USERNAME} LYX_DIR="${LYX_DIR}" bash -c 'source /etc/environment && python3 -tt "${LYX_DIR}/lib/configure.py" > ../"build-logs/reconfigure_lt" 2>&1' \
+    || { echo "$(ERRORPREFIX)reconfigure failed." >&2; mainRet=1; }
+)
+# todo: The following is copied from above. Instead, do a loop?
+# Can test that this worked by running "mylyx master-gcc" and trying to compile
+# a simple document to PDF.
+(
+  cd "${LYX_DIR_GCC}/../user-dir"
+  sudo -u ${USERNAME} LYX_DIR_GCC="${LYX_DIR_GCC}" bash -c 'source /etc/environment && python3 -tt "${LYX_DIR_GCC}/lib/configure.py" > ../"build-logs/reconfigure_lt" 2>&1' \
+    || { echo "$(ERRORPREFIX)reconfigure failed for GCC." >&2; mainRet=1; }
+)
+
+# Set up polyglossia-dev for use with "switch-polyglossia-to-dev"
+# this dir location should be synced with the switch functions in switches
+polyglossia_dev_d="/home/${USERNAME}/polyglossia"
+(
+  sudo -u ${USERNAME} git clone "https://github.com/reutenauer/polyglossia" "${polyglossia_dev_d}"
+)
+
+latex_dev_d="/home/${USERNAME}/latex2e-dev"
+(
+  sudo -u ${USERNAME} git clone "https://github.com/latex3/latex2e" "${latex_dev_d}"
+  cd "${latex_dev_d}" && sudo -u ${USERNAME} git checkout "develop"
+)
+
+if [[ "$mainRet" = "0" && "${RUN_TESTS}" = "1" ]]; then
+  echo "$(ECHOPREFIX)$(date +%H:%M): running ctest..."
+  if [ -e "${mainDir}/.lyx" ]; then
+    # Force a reconfigure.
+    rm -r "${mainDir}/.lyx"
+  fi
+  # LyX will not make the dir automatically when set with an environment variable.
+  sudo -u ${USERNAME} mkdir "${mainDir}/.lyx"
+  cd "${LYX_DIR}/../CMakeBuild" || { echo "$(ERRORPREFIX)could not cd into LyX CMake build dir" >&2; mainRet=1; }
+  # TODO
+  #  tex2lyx will fail if run first because tex2lyx requires a configured user-dir. See #8539
+  #  and
+  #  http://comments.gmane.org/gmane.editors.lyx.devel/144638
+  #  As a workaround, we run first the other tests first and then the tex2lyx tests.
+  # ALTERNATIVE: run one of the export tests then run -R '(export|tex2lyx)/'.
+  echo -e "$(ECHOPREFIX)PATH is:\n$PATH" >&3
+  # But just in case the dependencies are installed on a server, we do not want to run them (unless explicitly specified by autotests).
+  # We run them first to get them out of the way (so that the user knows they can use the machine without worry of confusing keytest).
+  if [ "${autotests}" = "1" ]; then
+    # XVKBD_DELAY overrides the def_delay default of 100. Only 120 was needed in my testing, but just in case I set it to 150.
+    sudo -u ${USERNAME} bash -c "source /etc/environment && DISPLAY=:0.0 XVKBD_DELAY=150 ctest --timeout ${CTEST_TIMEOUT} -R 'keytest/' > \"${LOGDIR}/ctest_keytest.log\" 2>&1"
+    cp "${LYX_DIR}/../CMakeBuild/Testing/Temporary/LastTest.log" "${LOGDIR}/LastTest_keytest.log"
+  fi
+
+  # TODO: the following two solutions (the one commented out and the one
+  # actually used) should work in theory, but they sometimes lead to the module
+  # ctests failing. Note that the modules are not copied over to the "layouts"
+  # subdirectory of the user directory, although I'm not sure that's the root
+  # cause of the problem.
+  #
+  # alternative:
+  # We could reconfigure manually, since lyxbuild was run before some
+  # dependencies (e.g., lilypond) were installed, and since lyxbuild runs a
+  # ctest (it is now a feature of lyxbuild).
+  # ( cd Testing/.lyx/ && sudo -u ${USERNAME} LYX_DIR="${LYX_DIR}" LOGDIR="${LOGDIR}" bash -c 'source /etc/environment && python3 -tt "${LYX_DIR}/lib/configure.py" > "${LOGDIR}/reconfigure_b4_ctest.log" 2>&1' \
+  #  || { echo "$(ERRORPREFIX)reconfigure failed." >&2; mainRet=1; } )
+  #
+  # mv instead of rm incase we want to see differences in configure logs
+  mv Testing/.lyx/ /tmp/DOTlyx1 && sudo -u ${USERNAME} mkdir Testing/.lyx
+  sudo -u ${USERNAME} bash -c "source /etc/environment && ctest -R 'examples/Welcome_pdf2'"
+  ## DBG for comparing
+  #cp -r Testing/.lyx /tmp/DOTlyx2
+  # Enable needauth for knitr, Sweave, gnuplot, etc
+  sudo -u ${USERNAME} bash -c "source /etc/environment && ./prefTest.pl test"
+
+  sudo -u ${USERNAME} bash -c "source /etc/environment && ctest -j${numCores} --timeout ${CTEST_TIMEOUT} -E 'tex2lyx/|keytest/' > \"${LOGDIR}/ctest_other.log\" 2>&1"
+  cp "${LYX_DIR}/../CMakeBuild/Testing/Temporary/LastTest.log" "${LOGDIR}/LastTest_other.log"
+  sudo -u ${USERNAME} bash -c "source /etc/environment && ctest --timeout ${CTEST_TIMEOUT} -R 'tex2lyx/' > \"${LOGDIR}/ctest_tex2lyx.log\" 2>&1"
+  cp "${LYX_DIR}/../CMakeBuild/Testing/Temporary/LastTest.log" "${LOGDIR}/LastTest_tex2lyx.log"
+  # TODO test make targets. Which others?
+  echo "$(ECHOPREFIX)TODO: doyxdoc gives a lot of warnings (which I currently redirect to the log). Fix them?"
+  sudo -u ${USERNAME} make doxydoc > "${LOGDIR}/doxydoc" 2>&1 || echo "$(WARNINGPREFIX)could not build doxydoc target"
+
+
+  echo "$(ECHOPREFIX)now we will run systemF tests with polyglossia-dev" >&3
+  (
+    # do a git pull to catch any recent commits (the previous tests can take a long time to run)
+    cd "${polyglossia_dev_d}" && sudo -u ${USERNAME} git pull
+    # bring in the switch functions
+    source "${mainDir}/switches"
+    switch-polyglossia-to-dev
+    cd "${LYX_DIR}/../CMakeBuild" &&
+      sudo -u ${USERNAME} bash -c "source /etc/environment && ctest -j${numCores} --timeout ${CTEST_TIMEOUT} -R "systemF" > \"${LOGDIR}/ctest_other_poly-dev.log\" 2>&1"
+  )
+fi
+
+if [ "${mainRet}" = "0" ]; then
+  echo "$(ECHOPREFIX)$(date +%H:%M): finished (all successful)."
+else
+  echo "$(ERRORPREFIX)$(date +%H:%M): finished (not all successful). Please report warnings and errors." >&2
+fi
+
+
+chmod 666 "${LOGDIR}"/*

development/lyx-tester/lyxbuild

@@ -0,0 +1,780 @@
+#!/usr/bin/env bash
+
+
+# File: lyxbuild
+
+# Run the following command for usage and description:
+# ./lyxbuild --help
+
+
+# Copyright 2013, Scott Kostyshak
+# Copyright 2013, Kornel Benko
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+set -o nounset
+
+function checkTerm() {
+  # check if the column is 0 and if not start a new line (useful bc of the progress bar).
+  # based on code from: http://stackoverflow.com/questions/2575037/how-to-get-the-cursor-position-in-bash
+  exec < /dev/tty
+  oldstty=$(stty -g)
+  stty raw -echo min 0
+  # on my system, the following line can be replaced by the line below it
+  echo -en "\033[6n" > /dev/tty
+  # tput u7 > /dev/tty    # when TERM=xterm (and relatives)
+  IFS=';' read -r -d R -a pos
+  stty $oldstty
+  # change from one-based to zero based so they work with: tput cup $row $col
+  row=$((${pos[0]:2} - 1))    # strip off the esc-[
+  col=$((${pos[1]} - 1))
+  if [ "${col}" = "0" ]; then
+    exit
+  else
+    echo "\n"
+  fi
+}
+
+function ECHOPREFIX() {
+  if [[ "$-" =~ i ]]; then
+    exit
+  else
+    echo -e "$(checkTerm)$( basename "$0" ): "
+  fi
+}
+
+function ERRORPREFIX() {
+  echo "$(ECHOPREFIX)ERROR: "
+}
+
+function WARNINGPREFIX() {
+  echo "$(ECHOPREFIX)WARNING: "
+}
+
+
+ARGS=$( getopt -o "j:ahr:vu:p:q:fogx" -l "jobs:,automake,help,profile:,version,user-dir:,patches-dir:,qt-build-dir:,qt5,patch-old,gcc,enable-export-tests" \
+      -n "$( basename "$0" )" -- "$@" )
+# If bad arguments, exit (getopt will have already written the error to STDERR).
+[ "$?" = "0" ] || exit 1
+
+eval set -- "$ARGS"
+
+automake=0
+help=0
+version=0
+proFile=
+njobs=1
+# clang is default
+# setting --gcc changes clang to 0.
+clang=1
+userUserDir=
+patchesDir=
+qtBuildDir=
+# for cmake:
+qt_version='QT6'
+# for autotools:
+configure_options='--enable-qt6'
+patch_old=0
+# 1 means enable it depending on if we are in a bisect.
+always_enable_export_tests=1
+
+while true
+do
+  case "$1" in
+  -j|--jobs)  njobs="$2"
+      if [[ $2 =~ ^[0-9]+$ ]]; then
+        njobs="$2"
+      else
+        echo "$(ERRORPREFIX)the argument to $1 must be a positive number." >&2
+        exit 1
+      fi
+      shift 2
+      ;;
+  -a|--automake)  automake=1
+      shift 1
+      ;;
+  -f|--qt5)  qt_version='QT5'
+      configure_options='--enable-qt5'
+      shift 1
+      ;;
+  -u|--user-dir)
+      if [ -d "$2" ]; then
+        userUserDir=$( readlink -f "$2" )
+      else
+        echo "$(ERRORPREFIX)the argument to $1 must be a folder that exists." >&2
+        exit 1
+      fi
+      shift 2
+      ;;
+  -g|--gcc)  clang=0
+      shift 1
+      ;;
+  -h|--help)  help=1
+      shift 1
+      ;;
+  -v|--version)  version=1
+      shift 1
+      ;;
+  -p|--patches-dir)
+      if [ -d "$2" ]; then
+        patchesDir=$( readlink -f "$2" )
+      else
+        echo "$(ERRORPREFIX)the argument to $1 must be a folder that exists." >&2
+        exit 1
+      fi
+      shift 2
+      ;;
+  -q|--qt-build-dir)
+      if [ -d "$2" ]; then
+        qtBuildDir=$( readlink -f "$2" )
+      else
+        echo "$(ERRORPREFIX)the argument to $1 must be a folder that exists." >&2
+        exit 1
+      fi
+      if [ ! -d "$2/qtbase/bin" ]; then
+        # simple check that the build directory is valid
+        echo "$(ERRORPREFIX)the argument to $1 must contain the subdirectory qtbase/bin" >&2
+        exit 1
+      fi
+      shift 2
+      ;;
+  -r|--profile)
+      if [ -f "$2" -a ! -w "$2" ]; then
+        echo "$(ERRORPREFIX)the file $2 is not writable" >&2
+        exit 1
+      elif [ ! -w "$( dirname "$2" )" ]; then
+        echo "$(ERRORPREFIX)the directory of $2 is not writable" >&2
+        exit 1
+      else
+        proFile="$( readlink -f "$2" )"
+      fi
+      shift 2
+      ;;
+  -o|--patch-old)  patch_old=1
+      shift 1
+      ;;
+  -x|--enable-export-tests)
+      # TODO: sanity check the possible values of the argument.
+      always_enable_export_tests=1
+      shift 1
+      ;;
+  --)
+      shift 1
+      break;;
+  *)
+      echo -e "$(ERRORPREFIX)the getopt command must be out of sync. Please report this."\\n\
+          "The following flag is not recognized: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [ "$#" != "0" ]; then
+  echo "$(ERRORPREFIX)the following arguments are not recognized: $@" >&2
+  exit 1
+fi
+
+if [ "${help}" = "1" ]; then
+  echo -e "
+    USAGE
+    lyxbuild [OPTIONS]
+
+    DESCRIPTION
+    lyxbuild builds LyX from a git checkout. CMake (default) or Automake can be
+    used and logs of each step in the build process are stored. A separate user
+    directory is set up in the parent folder. One sensible directory configuration
+    is to have something like ~/lyxbuilds/master/repo. Then, lyxbuild will create
+    ~/lyxbuilds/master/build-logs and ~/lyxbuilds/master/user-dir. To run this
+    LyX build, copy the functions 'mylyx' and 'mylyxcd' into your .bashrc (or
+    source it). You can then enter 'mylyx master' to run this build of the above
+    example. 'mylyx' finds the CMake or Automake binary and runs it with the
+    correct user-dir. 'mylyxcd' changes to the Git repo of the specified build.
+    They come with tab completion.
+
+    OPTIONS
+    -j N, --jobs N              \e[00;36m number of jobs to pass to make. Default is 1.                           \e[00m
+    -a, --automake              \e[00;36m use Automake instead of CMake.                                          \e[00m
+    -c, --clang                 \e[00;36m compile with Ubuntu's default location of clang.                        \e[00m
+    -f, --qt5                   \e[00;36m build with Qt 5.                                                        \e[00m
+    -p DIR, --patches-dir DIR   \e[00;36m path to dir containing patches to apply                                 \e[00m
+    -o, --patch-old             \e[00;36m apply needed patches to build old commits on new systems                \e[00m
+    -q DIR, --qt-build-dir DIR  \e[00;36m path to dir the build of Qt 5 to prefix to PATH                         \e[00m
+    -r FILE, --profile FILE     \e[00;36m path to (new or existing) file to append profiling info.                \e[00m
+    -h, --help                  \e[00;36m display this help output and exit.                                      \e[00m
+    -u DIR, --user-dir DIR      \e[00;36m the user-dir from which to copy.                                        \e[00m
+    -v, --version               \e[00;36m display version and contact info.                                       \e[00m
+"
+  exit 0
+fi
+
+if [ "${version}" = "1" ]; then
+  echo -e "\\nlyxbuild\\nVersion 0.3dev\\nAuthors: Scott Kostyshak, Kornel Benko\\nContact: skostysh@princeton.edu"
+  exit 0
+fi
+
+# automake's analog to CMake's -DLYX_STDLIB_DEBUG=ON
+configure_options="${configure_options} --enable-stdlib-debug=yes"
+
+if [ -n "${qtBuildDir}" ]; then
+  # Should I remove the qt5-stable PATH? I do not want it to be found if something is not found in
+  # qt5-dev (I would prefer to force an error). That would be invasive and fragile though. Maybe
+  # better to check that all of qt5-dev is found.
+  echo "export PATH=${qtBuildDir}/qtbase/bin:${PATH}"
+  export PATH="${qtBuildDir}/qtbase/bin:${PATH}"
+  if [ "${automake}" = "1" ]; then
+    export MYQTDIR="${qtBuildDir}/qtbase"
+    configure_options="${configure_options} --with-qt-dir=$MYQTDIR"
+  fi
+fi
+
+# lyxbuild should only be run in the LyX source directory
+# Otherwise, if it is accidentally run in a different dir with Git, "git clean"
+# could remove files unexpectedly.
+if [ ! -f 'lyx.1in' ]; then
+  echo "$(ERRORPREFIX)Root of LyX source directory not detected (no file 'lyx.1in')" >&2
+  exit 1
+fi
+
+# if not in a git directory, this will fail.
+echo "$(ECHOPREFIX)running git clean..."
+git clean -xdf > /dev/null 2>&1 \
+  || { echo "$(ERRORPREFIX)failed to run git clean. The git directory is required for now." >&2; exit 1; }
+
+if [ -n "${patchesDir}" ]; then
+  echo "$(ECHOPREFIX)applying patches..."
+  for patch in ${patchesDir}/*; do
+    # gitHash="$(head -n 1 "${patch}" | sed 's/From \([^ ]*\).*/\1/')"
+    # TODO checking subject line is weak. User might amend the patch it locally and not copy to the patches folder. On the other hand, this behavior could be desired.
+    subjectLine=$(head -n 4 "${patch}" | tail -n 1 | sed 's/Subject: \[PATCH[^\]*\] //')
+    # TODO change for 2.0.x...
+    if git log origin/master..master | grep "${subjectLine}"; then
+      echo "$(ECHOPREFIX)patch already applied, skipping ${patch}."
+    else
+      git am "${patch}" || { echo "$(ERRORPREFIX)failed to apply $(basename "${patch}"). Running git am --abort and exiting." >&2; git am --abort; exit 1; }
+    fi
+  done
+fi
+
+
+# https://stackoverflow.com/questions/70861523/check-if-git-bisect-is-active-in-a-script-in-a-robist-stable-way
+if git bisect log > /dev/null 2>&1; then
+  echo "$(ECHOPREFIX)detected git bisect in progress"
+  bisect=1
+else
+  bisect=0
+fi
+
+
+if [ "${bisect}" = 1 ]; then
+  # might as well just set automatically so I don't have to remember.
+  patch_old=1
+fi
+
+
+# It is empty if not on a named branch.
+# (https://stackoverflow.com/questions/6245570/how-do-i-get-the-current-branch-name-in-git)
+current_branch="$( git branch --show-current )"
+
+if [ "${current_branch}" = "master" ]; then
+  # copied from lyxrm_final_stages
+  if grep -R '\\origin unavailable' lib/doc lib/examples lib/templates; then
+    echo "ERROR: found instance(s) of \\origin unavailable"
+    echo "need to clean these up for Windows build. See the following for more information:"
+    echo "https://www.mail-archive.com/search?l=mid&q=ndao0s%24vvb%242%40ger.gmane.org"
+    # e.g., Message-ID:<20230105150956.b4a2vhy2lefb7sek@gary>
+    # (e.g., koma-book.bib is not found in the KOMA-Script_Book.lyx tests.
+    echo "We also use \\origin for ctests to figure out where to copy file dependencies from."
+    exit 1
+  fi
+fi
+
+
+if [ "${patch_old}" = "1" ]; then
+  # e.g., on 2.3.0 it will correctly compile with Qt5 (we did not have support
+  # for Qt6 at this point).
+  #
+  # "AUTO" is a reasonable choice. But sometimes want more control
+  # (e.g., a certain commit doesn't compile with the AUTO's choice).
+  qt_version="QT5"
+  echo "$(ECHOPREFIX)Compiling with Qt ${qt_version}..."
+
+  if [ "${bisect}" = "1" ]; then
+    hash_orig="$( git rev-parse HEAD )"
+  fi
+
+  echo "$(ECHOPREFIX)cherry-picking potentially needed fixes for compiling this older commit..."
+
+  # TODO: make an array of commits and loop through them. If they are not an
+  # ancestor, apply. However, need to give multiple commits for branches (e.g.,
+  # 2.3.x, 2.4.x, master)
+
+  cmake_major="$( cmake -E capabilities | jq .version.major )"
+  cmake_minor="$( cmake -E capabilities | jq .version.minor )"
+  # Criteria: CMake version 3.18 or greater (i.e., Ubuntu 21.04).
+  # This fix is not needed for CMake 3.13, but not sure for 3.14 <= x < 3.18.
+  if [ "${cmake_major}" -ge "3" ] && [ "${cmake_minor}" -ge "18" ] &&
+     # for now, enable only for 2.3.1 and before AND before about same time on master.
+     # This fix can be extended.
+     ! git merge-base --is-ancestor 2.3.1 HEAD &&
+     ! git merge-base --is-ancestor 305d449a HEAD; then
+    echo "applying commits needed for newer CMake versions"
+    # not sure all of these are needed.
+    # --keep-redundant-commits causes "git cherry pick" to succeed even if
+    # the cherry-pick leads to an empty change (otherwise non-zero exit).
+    if ! git merge-base --is-ancestor "1a440494" HEAD; then
+      git cherry-pick -x --keep-redundant-commits --strategy=recursive -X theirs 1a440494
+    fi
+    if ! git merge-base --is-ancestor "889d10cd" HEAD; then
+      git cherry-pick -x --keep-redundant-commits --strategy=recursive -X theirs 889d10cd
+    fi
+    if ! git merge-base --is-ancestor "80b3adc4" HEAD; then
+      git cherry-pick -x --keep-redundant-commits --strategy=recursive -X theirs 80b3adc4
+    fi
+    git checkout 2.3.6 CMakeLists.txt &&
+    git commit -m "CMakeLists.txt from 2.3.6" &&
+    git cherry-pick -x --keep-redundant-commits 7b44280da4fc015a4a930c311cc6073cebacf738
+  fi
+
+
+  if ! git merge-base --is-ancestor "51cc8aa9" HEAD; then
+    git cherry-pick -x --keep-redundant-commits --strategy=recursive -X theirs 51cc8aa9
+  fi
+  if ! git merge-base --is-ancestor "1a440494" HEAD; then
+    git cherry-pick -x --keep-redundant-commits 1a440494
+  fi
+
+  # TODO: loop!
+  if ! git merge-base --is-ancestor "b73ab025" HEAD; then
+    # TODO: add "-X theirs" to all (and centralize options?)
+    # this particular instance of "-X theirs" is needed for b6cb557.
+    git cherry-pick --keep-redundant-commits -x -X theirs "b73ab025"
+  fi
+
+  if ! git merge-base --is-ancestor "1ef1808c" HEAD; then
+    git cherry-pick --keep-redundant-commits -x -X theirs "1ef1808c"
+  fi
+
+  if ! git merge-base --is-ancestor "9649e80c" HEAD; then
+    git cherry-pick --keep-redundant-commits -x -X theirs "9649e80c"
+  fi
+
+  if ! git merge-base --is-ancestor "9715d350" HEAD; then
+    git cherry-pick --keep-redundant-commits -x -X theirs "9715d350"
+  fi
+
+  if ! git merge-base --is-ancestor  "8c4d6bbb" HEAD; then
+    git cherry-pick --keep-redundant-commits -x -X theirs "8c4d6bbb"
+  fi
+
+  if ! git merge-base --is-ancestor  "6874c86c" HEAD; then
+    git cherry-pick --keep-redundant-commits -x -X theirs "6874c86c"
+  fi
+
+  # EXPIRE:
+  # The following is not strictly (because we do not run the cmake dev flags if
+  # --patch-old) necessary and it sync not apply cleanly in many cases.
+  #if ! git merge-base --is-ancestor  "8d79860e" HEAD; then
+  #  git cherry-pick --keep-redundant-commits -x -X theirs "8d79860e"
+  #fi
+
+  # if 2.3.0 < HEAD < 2.3.x, ...
+  if git merge-base --is-ancestor 2.3.0 HEAD &&
+     ! git merge-base --is-ancestor 2.3.x HEAD; then
+       # Needed because of Python fixes.
+       echo "checking out 2.3.x's configure and lyx2lyx scripts"
+       git checkout origin/2.3.x "lib/configure.py" "lib/lyx2lyx/" "lib/scripts/layout2layout.py"
+  fi
+
+  # Newer compiler versions are more strict
+  WERROR=0
+  # LyX 2.2.0 fails to compile on Ubuntu 19.04 with CXX11. The warning
+  # starts with the following:
+  #   undefined reference to `boost::re_detail_106000::perl_matcher...
+  # (An alternative would be to backport some commits)
+  USE_CXX11="OFF"
+
+else
+  USE_CXX11="ON"
+  # enable WERROR after some fixes on master.
+  # 2.3.x fails with WERROR=1 for example.
+  #
+  # TODO: I'm not actually sure which commit fixed it for Ubuntu 21.10.
+  # In the future, need to condition on Qt version and on gcc version.
+  # We really only want to turn this on anyway when compiling
+  # master or when bisecting when the warning first started occurring.
+  warning_fix_commit="874ae71f"
+  if git merge-base --is-ancestor "${warning_fix_commit}" HEAD; then
+    WERROR=1
+  else
+    WERROR=0
+  fi
+fi
+
+
+echo "$(ECHOPREFIX)removing previous build logs..."
+[ -d ../build-logs ] && rm -rf ../build-logs
+mkdir ../build-logs
+
+# Start timing here for profiling.
+begin_chain="$(date +%s)"
+gitRepoDIR="$( readlink -f ./ )"
+
+# We want to do this for autotools also so mylyx() doesn't run an old cmake build rather than a new autotools build.
+[ -d ../CMakeBuild ] && { echo "$(ECHOPREFIX)removing previous CMake build" && rm -rf ../CMakeBuild ;}
+
+echo "$(ECHOPREFIX)updating layouts in LyX's lib..."
+python3 development/tools/updatelayouts.py
+
+
+if [ "${clang}" = "1" ]; then
+  export CC="clang"
+  # TODO: add option to build with libc++ :
+  # export CXX="clang++ -stdlib=libc++"
+  export CXX="clang++"
+else
+  # didn't use to specify this "else" because GCC
+  # was default, but the system default might not
+  # be GCC so it's best to be explicit.
+  export CC="gcc"
+  export CXX="g++"
+fi
+
+
+if [ "${automake}" = "1" ]; then
+
+  if [ "${WERROR}" = "1" ]; then
+    # see "Some influential environment variables" in ./configure --help
+    export CXXFLAGS="-Werror"
+  fi
+
+  echo "$(ECHOPREFIX)running autogen..."
+  ./autogen.sh > ../build-logs/autogen 2>&1 \
+    || { echo "$(ERRORPREFIX)running autogen.sh" >&2; exit 1; }
+
+  echo "$(ECHOPREFIX)running configure..."
+  # echo "configure options are: ${configure_options}"
+  ./configure ${configure_options} > ../build-logs/configure 2>&1 \
+    || { echo "$(ERRORPREFIX)running configure" >&2; exit 1; }
+else
+  # this check needs to be done before cd'ing to build dir (or need to cd back)
+  # only compile with Enchant on Ubuntu >= 20.04 if we have the fix that
+  # supports Enchant 2
+  ENCHANT_FLAG="ON"
+  ubuntu_ver="$( lsb_release -sr )"
+  ubuntu_ver_yr="${ubuntu_ver:0:2}"
+  if [ "${ubuntu_ver_yr}" -ge "20" ]; then
+    enchant_commit="f1694db49681451f9b947f25238f8032769799f8"
+    if ! git merge-base --is-ancestor "${enchant_commit}" HEAD; then
+      echo "CMake flags: Not compiling with Enchant."
+      ENCHANT_FLAG="OFF"
+    fi
+  fi
+
+  HUNSPELL_FLAG="ON"
+  # could condition on CMake version.
+  # This one cherry-pick's clean, but we might also need to cherry-pick the one
+  # this one is based on and that does not cherry-pick clean.
+  hunspell_commit="cf980435b12e697b90508924cf0c82dd3c1c0b36"
+  if ! git merge-base --is-ancestor "${hunspell_commit}" HEAD; then
+      echo "CMake flags: Not compiling with Hunspell."
+    HUNSPELL_FLAG="OFF"
+  fi
+
+  # this commit is not the one that fixed it, it is just known to work.
+  if git merge-base --is-ancestor "3aab9ad2" HEAD; then
+    # -Werror=dev turns CMake warnings into errors, but not all warnings. For
+    # example, cannot turn warnings about unused flags into errors:
+    # https://stackoverflow.com/questions/42684572/how-to-make-cmake-exit-non-0-in-response-to-warning-manually-specified-variable
+    #
+    # -Wdev has the following documentation:
+    # Enable developer warnings.
+    # Enable warnings that are meant for the author of the CMakeLists.txt files.
+    #
+    cmake_Werror_flags='-Wdev -Werror=dev'
+  else
+    cmake_Werror_flags='-Wno-dev -Wno-error=dev'
+  fi
+  # for --patch-old, just ignore warnings:
+  if [ "${patch_old}" = "1" ]; then
+    cmake_Werror_flags='-Wno-dev -Wno-error=dev'
+  fi
+
+
+  echo "$(ECHOPREFIX)creating \"CMakeBuild\" dir and running cmake..."
+  mkdir ../CMakeBuild
+  cd ../CMakeBuild
+
+
+  if [ "${WERROR}" = "1" ]; then
+    # should be a semi-colon separated list (if more than one option).
+    CPPFLAGS="-Werror"
+    if [ "${clang}" = "0" ]; then
+      # (https://stackoverflow.com/questions/5188267/checking-the-gcc-version-in-a-makefile)
+      # (https://stackoverflow.com/a/17947005)
+      gcc_version="$( gcc -dumpfullversion -dumpversion | sed -e 's/\.\([0-9][0-9]\)/\1/g' -e 's/\.\([0-9]\)/0\1/g' -e 's/^[0-9]\{3,4\}$/&00/')"
+      if [ "${gcc_version}" -ge "130000" ]; then
+        # turn all warnings into errors except for one:
+        #
+        # 'dangling-reference' is starting with GCC 13.
+        # In LyX, we get it in a few places and are not sure if they are
+        # false-positives, or if it's worth it to fix them.
+        CPPFLAGS+=";-Wno-error=dangling-reference"
+      fi
+    fi
+  else
+    CPPFLAGS=""
+  fi
+  #
+  # Should be a semi-colon separated list (if more than one option).
+  # The following naive Bash append works well even if CPPFLAGS is empty,
+  # i.e., ";-O3" is processed well.
+  CPPFLAGS_FAST="${CPPFLAGS};-O3;-march=native"
+
+  if [ "${always_enable_export_tests}" = "1" ]; then
+    ctest_flags="ON"
+  else
+    if [ "${bisect}" = "1" ]; then
+      # This flag just controls whether cmake configures the tests,
+      # which can take e.g. a minute for the export tests.
+      # This flag does not control whether the tests are actually run.
+      ctest_flags="OFF"
+    else
+      ctest_flags="ON"
+    fi
+  fi
+
+  cmake_flags=(
+    # TODO what about /usr/local/git_branch ?
+    -DCMAKE_INSTALL_PREFIX=/usr/local
+    -DLYX_DEBUG=ON -DLYX_RELEASE=OFF -DLYX_CPACK=ON
+    # set to OFF so binary now has name "lyx" instead of e.g. "lyx2.3".
+    # I like this so that if I install, all my scripts can just call "lyx"
+    # todo: but might be easy to make a separate 'lyx' binary that just
+    # points to one lyx2.3 or other.
+    -DLYX_PROGRAM_SUFFIX=OFF
+    -DLYX_LOCALVERSIONING=ON
+    # note, these all depend on the option -DLYX_CPACK=ON
+    # (otherwise unrecognized)
+    -DCPACK_BINARY_DEB:BOOL=ON
+    # requires "rpmbuild" binary from Ubuntu "rpm" package.
+    -DCPACK_BINARY_RPM:BOOL=OFF
+    -DCPACK_BINARY_NSIS:BOOL=OFF -DCPACK_BINARY_STGZ:BOOL=OFF
+    # set to ON for "make package" to create a tar file (of the *binaries*)
+    # (not sure if useful)
+    -DCPACK_BINARY_TGZ:BOOL=OFF
+    -DCPACK_BINARY_TBZ2:BOOL=OFF
+    -DCPACK_BINARY_TZ:BOOL=OFF
+    # These control behavior of "make package_source"
+    -DCPACK_SOURCE_TGZ:BOOL=ON
+    -DCPACK_SOURCE_TBZ2:BOOL=OFF
+    -DCPACK_SOURCE_TZ:BOOL=OFF
+    -DCPACK_SOURCE_ZIP:BOOL=OFF -DLYX_EXTERNAL_BOOST=OFF
+    -DLYX_HUNSPELL=${HUNSPELL_FLAG}
+    # if LYX_EXTERNAL_HUNSPELL=ON and LYX_DEBUG_GLIBC=ON LyX will crash when
+    # correcting a word with hunspell as spellchecker.
+    # See d9e0a842 and bda9e38a.
+    # Note that if we mistakenly turn both on, CMake catches it during the
+    # 'cmake' command.
+    # TODO: Before Ubuntu 21.10, I would set EXTERNAL_HUNSPELL to OFF
+    # and STDLIB_DEBUG=OFF to ON. This now causes LyX to crash when
+    # correcting a word.
+    -DLYX_EXTERNAL_HUNSPELL=${HUNSPELL_FLAG}
+    -DLYX_ENCHANT=${ENCHANT_FLAG}
+    -DLYX_EXTERNAL_Z=ON
+    -DLYX_EXTERNAL_DTL=ON
+    -DLYX_EXTERNAL_ICONV=ON
+    -DLYX_NLS=ON
+    -DLYX_ENABLE_URLTESTS=OFF
+    -DLYX_ENABLE_EXPORT_TESTS=${ctest_flags}
+    -DLYX_ENABLE_KEYTESTS=OFF
+    -DLYX_PROFILE=ON
+    -DLYX_USE_QT=${qt_version} -DLYX_CXX_FLAGS_EXTRA="${CPPFLAGS}"
+    # Compile with STL debug code.
+    # For one particular use case, see:
+    #   https://www.lyx.org/trac/ticket/11204
+    # see https://www.lyx.org/trac/ticket/11777#comment:10
+    #  -DLYX_DEBUG_GLIBC=ON
+    # see d9e0a842
+    # STDLIB_DEBUG=ON is possibly helpful for getting better backtraces
+    #   (https://www.lyx.org/trac/ticket/12226#comment:28)
+    -DLYX_STDLIB_DEBUG=OFF
+    -DLYX_DEBUG_GLIBC=OFF
+    -DLYX_DEBUG_GLIBC_PEDANTIC=OFF
+    #-DLYX_DEBUG_SANITIZE="ADDRESS"
+    # TODO: enable only for GCC? Clang currently gives output that
+    #       I'm not sure how to fix and not sure if it's a real issue.
+    #-DLYX_DEBUG_SANITIZE="UNSPECIFIED"
+  )
+
+  cmake_flags_fast=(
+    -DCMAKE_INSTALL_PREFIX=/usr/local
+    -DLYX_DEBUG=OFF -DLYX_RELEASE=ON -DLYX_CPACK=OFF
+    -DLYX_PROGRAM_SUFFIX=OFF
+    -DLYX_LOCALVERSIONING=ON
+    -DLYX_EXTERNAL_BOOST=OFF
+    -DLYX_HUNSPELL=${HUNSPELL_FLAG}
+    -DLYX_EXTERNAL_HUNSPELL=${HUNSPELL_FLAG}
+    -DLYX_ENCHANT=${ENCHANT_FLAG}
+    -DLYX_EXTERNAL_Z=ON
+    -DLYX_EXTERNAL_DTL=ON
+    -DLYX_EXTERNAL_ICONV=ON
+    -DLYX_NLS=ON
+    -DLYX_ENABLE_URLTESTS=OFF
+    -DLYX_ENABLE_EXPORT_TESTS=${ctest_flags}
+    -DLYX_ENABLE_KEYTESTS=OFF
+    -DLYX_PROFILE=OFF
+    -DLYX_USE_QT=${qt_version} -DLYX_CXX_FLAGS_EXTRA="${CPPFLAGS_FAST}"
+    -DLYX_STDLIB_DEBUG=OFF
+    -DLYX_DEBUG_GLIBC=OFF
+    -DLYX_DEBUG_GLIBC_PEDANTIC=OFF
+  )
+
+  cmake ${cmake_Werror_flags} "${gitRepoDIR}/" ${cmake_flags[@]} > ../build-logs/cmake 2>&1 \
+    || { echo "$(ERRORPREFIX)running cmake" >&2; exit 1; }
+fi
+
+echo "$(ECHOPREFIX)running make..."
+echo "$(ECHOPREFIX)building with ${njobs} core(s)" > ../build-logs/make
+make -j${njobs} >> ../build-logs/make 2>&1
+make_ret=$?
+if [ "${make_ret}" != 0 ]; then
+  echo "$(ERRORPREFIX)running make" >&2
+  cd "${gitRepoDIR}"
+  if [ "${bisect}" = "1" ] && [ "${patch_old}" = "1" ]; then
+    # The following is helpful when we are bisecting compilation success.
+    # todo: but it might be annoying when we are not, because we might want to
+    #       cd and then manually run 'make' to see the error, but because of
+    #       this it could cause a different error.
+    #
+    # restore so that "git bisect good/bad" works.
+    echo "$(ECHOPREFIX)reseting and checking out so that good/bad works..."
+    git reset --hard
+    git checkout "${hash_orig}"
+  fi
+  exit 1
+fi
+end_chain=$(date +%s)
+if [ "${patch_old}" != "1" ]; then
+  # extra checks in make log.
+  #
+  # Check for warnings from msgmerge (see e.g., the LyX commit 46aedb4e).
+  # msgmerge does not seem to have an option to convert warnings to errors
+  # (--strict does not do it). We could instead propose to write a wrapper script
+  # for msgmerge that checks for this output (i.e., upstream this check to theLyX
+  # build scripts).
+  if grep "warning: internationalized messages should not contain" ../build-logs/make; then
+    echo "$(ERRORPREFIX)invalid string(s) in po file(s). See grepped lines just above." >&2
+    exit 1
+  fi
+fi
+if [ "${automake}" = "0" ]; then
+  # Can be any test. We have to run some test though, so that the ctest user
+  # directory is configured so that prefTest (see below) can be run.
+  ctest -R "examples/Welcome_pdf2"
+  # Enable needauth for knitr, Sweave, gnuplot, etc for ctests.
+  # This enables it only for the ctest user directory.
+  ./prefTest.pl test
+fi
+
+DIFF=$(( end_chain - begin_chain ))
+cd "${gitRepoDIR}"
+
+compname=$( uname -n )
+commitinfo=$( git describe )
+
+MASTER_DIR=$( readlink -f ../ )
+LYX_USER_DIR="${MASTER_DIR}/user-dir"
+
+[ -d "${LYX_USER_DIR}" ] && rm -r "${LYX_USER_DIR}"
+mkdir "${LYX_USER_DIR}"
+
+if [ "${automake}" = "1" ]; then
+  echo "$(ECHOPREFIX)checking out 'po/'..."
+  git checkout "po/" ||
+    echo "$(ERRORPREFIX)checkout of 'po/' failed" >&2
+fi
+
+if [ -n "${proFile}" ]; then
+  if [ -f "${proFile}" ]; then
+    echo "${compname} ${commitinfo} ${njobs} ${DIFF} ${automake}" >> "${proFile}"
+  else
+    echo "compname commitinfo njobs DIFF automake" > "${proFile}"
+    echo "${compname} ${commitinfo} ${njobs} ${DIFF} ${automake}" >> "${proFile}"
+  fi
+fi
+
+cd "${LYX_USER_DIR}"
+if [ -n "${userUserDir}" ]; then
+  echo "$(ECHOPREFIX)copying user-dir..."
+  cp -r "${userUserDir}"/* ./
+  # LyX does not currently allow ~ in session files,
+  # so need to substitute the user name in.
+  sed -i "s/\/home\/[^/]*/\/home\/${USER}/g" session
+  #
+  # resolve lnx files
+  # Because many services (like Dropbox) do not support links,
+  # can use a manually-implemented link. A .lnx file is just
+  # a plain-text file with one line that is the path to
+  # a different file.
+  # Here, we convert that to an actual symbolic link.
+  # TODO: allow to use .lnx files besides just layouts ? e.g., preferences?
+  if [ -d "layouts" ]; then
+    shopt -s nullglob
+    for lnx in layouts/*.lnx; do
+      sed -i "s/~/\/home\/${USER}/g" "${lnx}"
+      pathtofile="$( cat "${lnx}" )"
+      if [ -e "${pathtofile}" ]; then
+        ln -s "${pathtofile}" "${lnx%.*}"
+      fi
+    done
+    shopt -u nullglob
+  fi
+fi
+
+# Just for convenience (of not having to configure when starting LyX and of having preferences)
+echo "$(ECHOPREFIX)reconfiguring..."
+# LyX did not have perfect Python3 support for a while, so we use Python2 for
+# older commits.
+# This commit is conservative (could make it older).
+# The actual commit has nothing to do with Python3, we just use it
+# since it has been tested to work with Python3.
+cd "${gitRepoDIR}"
+python3_commit="af7ffc7dfa00297e58d332d57f4ead76265cdda3"
+if git merge-base --is-ancestor "${python3_commit}" HEAD; then
+  PYTHON_EXEC="python3"
+else
+  PYTHON_EXEC="python2"
+fi
+# TODO: expire the above. "python2" is no longer available Ubuntu releases
+#       starting with 23.04.
+PYTHON_EXEC="python3"
+
+cd "${LYX_USER_DIR}"
+${PYTHON_EXEC} -tt "${gitRepoDIR}/lib/configure.py" > ../build-logs/reconfigure 2>&1 \
+  || { echo "$(WARNINGPREFIX)reconfigure failed." >&2; exit 1; }
+
+LYX_USER_DIR_NOAUTH="${LYX_USER_DIR}-noauth"
+[ -d "${LYX_USER_DIR_NOAUTH}" ] && rm -r "${LYX_USER_DIR_NOAUTH}"
+cp -r "${LYX_USER_DIR}" "${LYX_USER_DIR_NOAUTH}"
+echo "\\use_converter_needauth false" >> "${LYX_USER_DIR_NOAUTH}"/preferences
+
+cd "${gitRepoDIR}"
+if [ "${bisect}" = "1" ] && [ "${patch_old}" = "1" ]; then
+  # restore so that "git bisect good/bad" works.
+  echo "$(ECHOPREFIX)checking out original bisect commit, before patches..."
+  git reset --hard
+  git checkout "${hash_orig}"
+fi
+echo "$(ECHOPREFIX)done."

development/lyx-tester/mylyx

@@ -0,0 +1,169 @@
+# set to 0 to disable.
+#
+# If set to 1 (or currently any non-zero value), mylyx will read the file and
+# open with the appropriate LyX version. If the format is less than the master
+# format, we don't open with master because we want to know when the file
+# format changes (and when to review lyx2lyx diff).
+_mylyx_veropen_=1
+# TODO: create and use a variable _mylyx_dir_
+# _mylyx_dir_="~/lyxbuilds/"
+
+
+function _validate_requested_mylyx ()
+{
+  potential_dir="/home/${USER}/lyxbuilds/${requested_build}"
+  if [ ! -d "${potential_dir}" ]; then
+    echo "ERROR: the requested mylyx build does not exist: ${potential_dir}" >&2
+    return 1
+  fi
+
+  return 0
+}
+
+
+
+function mylyx ()
+{
+  if [ "${_mylyx_veropen_}" = "0" ]; then
+    requested_build="$1"
+    _validate_requested_mylyx "$1" || return 1
+    shift
+  else
+    if [ -f "$1" ]; then
+      # TODO: need to fix tab expansion.
+      # we assume user wants the default build
+      lyxformat="$( head -n2 "$1" | grep -o -P "\d\d\d" )"
+      echo "lyxformat is: ${lyxformat}"
+      if [ "${lyxformat}" = "544" ]; then
+        requested_build="2.3.x"
+      elif [ "${lyxformat}" = "474" ]; then
+        requested_build="2.1.0"
+      elif [ "${lyxformat}" = "620" ]; then
+        requested_build="2.4.x"
+      else
+        # now we see what format my current master build is in
+        master_frmt="$( grep "LYX_FORMAT_LYX" ~/lyxbuilds/master/repo/src/version.h | grep -P -o "\d\d\d" )"
+        if [ "${lyxformat}" = "${master_frmt}" ]; then
+          requested_build="master"
+        else
+          echo "No build associated with LyX format ${lyxformat}." >&2
+          return 1
+        fi
+      fi
+      echo "Auto-selecting ${requested_build}"
+    else
+      requested_build="$1"
+      shift
+    fi
+
+    _validate_requested_mylyx "${requested_build}" || return 1
+  fi
+
+  if [ -e ~/lyxbuilds/$requested_build/repo/src/lyx ]; then
+    echo "starting local autotools binary"
+    # This assumes we're using --qt-build-dir when building. Otherwise,
+    # mismatch could happen.
+    # not sure why need to set LD_LIBRARY_PATH for autotools and not for CMake
+    # how does CMake find it?
+    LD_LIBRARY_PATH='/usr/BUILD/BuildQt5-dev/qtbase/lib' ~/lyxbuilds/$requested_build/repo/src/lyx -userdir ~/lyxbuilds/$requested_build/user-dir/ "$@"
+  else
+    # the regex "^lyx(\d\.\d)?" allows for both lyx2.3 and lyx. Depends on what the user sets for -DLYX_PROGRAM_SUFFIX
+    LYX_VER="$( ls ~/lyxbuilds/$requested_build/CMakeBuild/bin/ | grep -oP "^lyx(\d\.\d)?$" )"
+    if [ -e ~/lyxbuilds/$requested_build/CMakeBuild/bin/${LYX_VER} ]; then
+      echo "starting local cmake binary for ${LYX_VER}"
+      # Adding the build's bin dir to PATH makes it so e.g., the local build's
+      # tex2lyx is used, rather than a system-installed tex2lyx.
+      PATH="/home/${USER}/lyxbuilds/${requested_build}/CMakeBuild/bin:${PATH}" ~/lyxbuilds/$requested_build/CMakeBuild/bin/${LYX_VER} -userdir ~/lyxbuilds/$requested_build/user-dir/ "$@"
+    else
+      echo "ERROR: no autotools or cmake binary available" >&2
+    fi
+  fi
+}
+
+
+_mylyx()
+{
+  local cur
+
+  cur=${COMP_WORDS[COMP_CWORD]}
+
+  if [ "$COMP_CWORD" == "1" ]; then
+    COMPREPLY=( $( compgen -W '`ls ~/lyxbuilds`' -- $cur ) )
+
+    if [ "${_mylyx_veropen_}" != "0" ]; then
+      # need to protect against an empty directory, otherwise get the following
+      # error:
+      #   mylyx find: ‘*’: No such file or directory
+      if [ "$(ls -A .)" ]; then
+        # for find command:
+        #   https://stackoverflow.com/a/2596474/1376404
+        # for case insenstivity, replace ${cur} with ${cur,,}:
+        #   (this messed something else up so I changed back.
+        #   https://unix.stackexchange.com/a/128390/197212
+        #   This one is also useful:
+        #   https://stackoverflow.com/a/10981916/1376404
+        COMPREPLY+=( $( compgen -W '`find * -maxdepth 0 -type f -name "*.lyx"`' -- "${cur}" ) )
+      fi
+    fi
+  else
+    _filedir '@(lyx)'
+  fi
+}
+complete -F _mylyx ${filenames:-} mylyx
+
+
+function mylyxcd ()
+{
+  requested_build="$1"
+  _validate_requested_mylyx "$1" || return 1
+
+  # TODO could allow for 'repo' to be named anything. Just check for a git folder.
+
+  cd ~/lyxbuilds/$requested_build/repo
+}
+
+
+_mylyxcd()
+{
+  local cur
+
+  cur=${COMP_WORDS[COMP_CWORD]}
+
+  if [ "$COMP_CWORD" == "1" ]; then
+    COMPREPLY=( $( compgen -W '`ls ~/lyxbuilds`' -- $cur ) )
+  fi
+}
+complete -F _mylyxcd ${filenames:-} mylyxcd
+
+
+function mylyx-gdb ()
+{
+  # (https://stackoverflow.com/questions/9057387/process-all-arguments-except-the-first-one-in-a-bash-script)
+  # the "${@:2}" relays all arguments except the first.
+  requested_build="$1"
+  _validate_requested_mylyx "$1" || return 1
+  PATH="/home/${USER}/lyxbuilds/${requested_build}/CMakeBuild/bin:${PATH}" gdb --args ~/"lyxbuilds/${requested_build}/CMakeBuild/bin/lyx" -userdir ~/"lyxbuilds/${requested_build}/user-dir" "${@:2}"
+}
+complete -F _mylyxcd ${filenames:-} mylyx-gdb
+
+
+function mylyx-valgrind ()
+{
+  # (https://stackoverflow.com/questions/9057387/process-all-arguments-except-the-first-one-in-a-bash-script)
+  # the "${@:2}" relays all arguments except the first.
+  requested_build="$1"
+  _validate_requested_mylyx "$1" || return 1
+  PATH="/home/${USER}/lyxbuilds/${requested_build}/CMakeBuild/bin:${PATH}" valgrind --track-origins=yes --log-file=valgrind.log ~/lyxbuilds/${requested_build}/CMakeBuild/bin/lyx -userdir ~/"lyxbuilds/${requested_build}/user-dir" "${@:2}"
+}
+complete -F _mylyxcd ${filenames:-} mylyx-valgrind
+
+
+function mylyx-valgrind-leak-check-full ()
+{
+  # (https://stackoverflow.com/questions/9057387/process-all-arguments-except-the-first-one-in-a-bash-script)
+  # the "${@:2}" relays all arguments except the first.
+  requested_build="$1"
+  _validate_requested_mylyx "$1" || return 1
+  PATH="/home/${USER}/lyxbuilds/${requested_build}/CMakeBuild/bin:${PATH}" valgrind --track-origins=yes --log-file=valgrind.log --leak-check=full ~/lyxbuilds/${requested_build}/CMakeBuild/bin/lyx -userdir ~/"lyxbuilds/${requested_build}/user-dir" "${@:2}"
+}
+complete -F _mylyxcd ${filenames:-} mylyx-valgrind-leak-check-full

development/lyx-tester/ssh-lyx-tester

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+
+# File: ssh-lyx-tester
+
+# Usage:
+# ssh-lyx-tester <path-to-key> <server>
+
+# Description:
+# Uploads required files to a server and runs lyx-tester
+
+
+# Copyright 2013, Scott Kostyshak
+# Copyright 2013, Kornel Benko
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+set -o nounset
+
+if [[ "$-" =~ i ]]; then
+  ECHOPREFIX=
+else
+  ECHOPREFIX="$( basename "$0" ): "
+fi
+ERRORPREFIX="${ECHOPREFIX}Error: "
+WARNINGPREFIX="${ECHOPREFIX}Warning: "
+USAGE="correct usage: $( basename "$0" ) <key> <server>"
+
+# TODO implement command parsing and options.
+# if SOURCE_ENV is 1, then the path will be reloaded in each new bash instance.
+# This is useful on a server when you don't want to restart.
+SOURCE_ENV=0
+
+if [ "$#" != "2" ]; then
+  echo -e "${ERRORPREFIX}incorrect usage.\n${USAGE}"
+  exit 1
+fi
+
+key="$1"
+server="$2"
+
+if [ ! -r ${key} ]; then
+  echo -e "${ERRORPREFIX}specified key does not exist or does not have read permission.\n${USAGE}"
+  exit 1
+fi
+
+# This is a personal preference (if this line is removed though, you might need
+# to run the next ssh/scp command with -o StrictHostKeyChecking=no).
+ssh -o StrictHostKeyChecking=no -f -i "${key}" ubuntu@${server} 'byobu-launcher-install' \
+  >/dev/null 2>&1 || { echo "${ERRORPREFIX}could not run byobu-launcher-install" >&2; exit 1; }
+
+filesToTransfer=(
+  lyx-tester
+  debian-control-texlive-in.txt
+  install-tl-ubuntu
+  lyxbuild
+)
+
+for file in ${filesToTransfer[@]}; do
+  [ -e "${file}" ] || { echo "${ERRORPREFIX}required file, ${file}, does not exist." >&2; exit 1; }
+done
+
+echo "${ECHOPREFIX}transfering files..."
+scp -r -i "${key}" "${filesToTransfer[@]}" ubuntu@${server}:/home/ubuntu/ >/dev/null 2>&1 \
+  || { echo "${ERRORPREFIX}could not scp the main files over." >&2; exit 1; }
+echo "${ECHOPREFIX}transfer done."
+
+ssh -f -i "${key}" ubuntu@${server} "mkdir /home/ubuntu/lyx-tester-logs && sudo /home/ubuntu/lyx-tester >/home/ubuntu/lyx-tester-logs/mainlog.log 2>&1" \
+  || { echo "${ERRORPREFIX}could not dispatch lyx-tester." >&2; exit 1; }
+
+if [ "${SOURCE_ENV}" ]; then
+  ssh -i "${key}" ubuntu@${server} "echo \"source /etc/environment\" >> ~/.bashrc" \
+    || { echo "${ERRORPREFIX}: could not append to .bashrc." >&2; }
+fi
+
+echo "${ECHOPREFIX}lyx-tester is off and running on the remote server."

development/lyx-tester/switches

@@ -0,0 +1,211 @@
+
+# these functions are useful for switching between dev versions
+# and stable versions of certain components.
+
+# running a function when the state is already the version
+# the function switches to should be a no-op (with a diagnostic
+# message)
+
+function switch-polyglossia-to-dev ()
+{
+
+  # Udi pointed out a potential problem:
+  #
+  #   Note that for example you don't watch the files from polyglossia installed in
+  #   "fonts/misc/xetex/fontmapping/polyglossia/"
+  #   but they rarely get updated.
+  #
+  # todo: In these functions we could at least check whether those files in the
+  #       repo are different so we are notified when we need to update them.
+
+
+  # where to archive the TL version
+  polyglossia_dev_d="/home/$(logname)/polyglossia"
+
+  # location of TL version
+  polyglossia_tl_f="$(/opt/texbin/kpsewhich polyglossia.sty)"
+  polyglossia_tl_d="$(dirname "${polyglossia_tl_f}")"
+  # check whether link or not link to see whether current is tl or dev.
+  if [ -L "${polyglossia_tl_d}" ]; then
+    echo "Link exists, so you are already using polyglossia-dev."
+    return 0
+  else
+    # overwrite polyglossia_tl since newer stable version
+    sudo mv "${polyglossia_tl_d}" "/home/$(logname)/polyglossia_tl"
+    # Use a link to poly-dev. That is, do not copy the directory. That way, we
+    # can just do a git pull.
+    sudo ln -s "${polyglossia_dev_d}/tex" "${polyglossia_tl_d}"
+    # we need to check for possibly new/removed files
+    sudo /opt/texbin/mktexlsr
+
+    \sudo -k
+  fi
+}
+
+
+function switch-polyglossia-to-stable ()
+{
+  polyglossia_stable_d="/home/$(logname)/polyglossia_tl"
+  polyglossia_tl_f="$(/opt/texbin/kpsewhich polyglossia.sty)"
+  polyglossia_tl_d="$(dirname "${polyglossia_tl_f}")"
+  if [ -L "${polyglossia_tl_d}" ]; then
+    sudo rm "${polyglossia_tl_d}"
+    sudo mv "${polyglossia_stable_d}" "${polyglossia_tl_d}"
+    sudo /opt/texbin/mktexlsr
+
+    \sudo -k
+  else
+    echo "Link does not exist, so you are already using polyglossia-stable."
+    return 0
+  fi
+}
+
+
+# useful to see what the initial state is to know whether to switch back, e.g.,
+# in a script that updates the TeX Live installation.
+function polyglossia-is-stable ()
+{
+  polyglossia_stable_d="/home/$(logname)/polyglossia_tl"
+  polyglossia_tl_f="$(/opt/texbin/kpsewhich polyglossia.sty)"
+  polyglossia_tl_d="$(dirname "${polyglossia_tl_f}")"
+  if [ -L "${polyglossia_tl_d}" ]; then
+    # 0 = no, not stable
+    echo "0"
+  else
+    # 1 = yes, stable
+    echo "1"
+  fi
+  return 0
+}
+
+
+tl_stable_bu_d="/home/$(logname)/tl-stable-binaries"
+tl_texbin_d="/opt/texbin/"
+function switch-latex-to-tl-dev ()
+{
+  if [ ! -e "${tl_stable_bu_d}" ]; then
+    mkdir "${tl_stable_bu_d}"
+  fi
+
+  # todo: how to find this more generally? could take dir of 'which pdflatex'
+  # todo: use the "real" link instead? shouldn't make a difference right?
+  # $ readlink -f /opt/texbin
+  # /usr/local/texlive/2021/bin/x86_64-linux
+  for i in "${tl_texbin_d}"/*-dev; do
+    devname="$( basename "${i}")"
+    stablename="${devname%-*}"
+    if [ $(cat "${tl_texbin_d}/${stablename}" | wc -l) = "2" ]; then
+      echo "${stablename} already points to dev"
+    else
+      echo "${stablename} was calling stable."
+      sudo mv "${tl_texbin_d}/${stablename}" "${tl_stable_bu_d}"
+      # todo: confirm this?
+      # doesn't work. Maybe pdflatex (or pdftex rather) checks the binary name calling it (?)
+      # sudo ln -s "${tl_texbin_d}/${devname}" "${tl_texbin_d}/${stablename}"
+      echo "#!/usr/bin/bash" | sudo tee "${tl_texbin_d}/${stablename}" > /dev/null
+      # need to escape " and $ but not @.
+      echo "/opt/texbin/${devname} \"\$@\"" | sudo tee -a "${tl_texbin_d}/${stablename}" > /dev/null
+      sudo chmod +x "${tl_texbin_d}/${stablename}"
+      echo "  ${stablename} now pointing to dev."
+
+      \sudo -k
+    fi
+  done
+}
+
+
+function switch-latex-to-tl-stable ()
+{
+  for i in "${tl_texbin_d}"/*-dev; do
+    devname="$( basename "${i}")"
+    stablename="${devname%-*}"
+    # The file should contain just a shebang line and one line if it
+    # is pointing to dev. Otherwise should be a link.
+    if [ $(cat "${tl_texbin_d}/${stablename}" | wc -l) = "2" ]; then
+      echo "${stablename} was calling dev."
+      if [ ! -e "${tl_stable_bu_d}/${stablename}" ]; then
+        echo "  restoring stable."
+        sudo rm "${tl_texbin_d}/${stablename}"
+        sudo cp -d "${tl_stable_bu_d}/${stablename}" "${tl_texbin_d}/${stablename}"
+
+        \sudo -k
+      else
+        echo "ERROR: the stable backup file does not exist: ${tl_stable_bu_d}/${stablename}"
+      fi
+    else
+      echo "${stablename} is already stable version."
+    fi
+  done
+}
+
+
+function switch-gcc-to ()
+{
+  sudo update-alternatives --set gcc "/usr/bin/gcc-$1"
+  sudo update-alternatives --set g++ "/usr/bin/g++-$1"
+  sudo update-alternatives --set gfortran "/usr/bin/gfortran-$1"
+
+  \sudo -k
+}
+_switch_gcc_to()
+{
+  local cur
+
+  cur=${COMP_WORDS[COMP_CWORD]}
+
+  if [ "$COMP_CWORD" == "1" ]; then
+    gcc_vers="$(ls /usr/bin/gcc-* | grep -o -P '(?<=gcc-)\d+(?=$)')"
+    COMPREPLY=( $( compgen -W '${gcc_vers}' -- $cur ) )
+  fi
+}
+complete -F _switch_gcc_to ${filenames:-} switch-gcc-to
+
+
+function switch-clang-to ()
+{
+  sudo update-alternatives --set clang "/usr/bin/clang-$1"
+  sudo update-alternatives --set clang++ "/usr/bin/clang++-$1"
+  # I don't think Clang has a (built-in) fortran
+
+  \sudo -k
+}
+_switch_clang_to()
+{
+  local cur
+
+  cur=${COMP_WORDS[COMP_CWORD]}
+
+  if [ "$COMP_CWORD" == "1" ]; then
+    clang_vers="$(ls /usr/bin/clang-* | grep -o -P '(?<=clang-)\d+(?=$)')"
+    COMPREPLY=( $( compgen -W '${clang_vers}' -- $cur ) )
+  fi
+}
+complete -F _switch_clang_to ${filenames:-} switch-clang-to
+
+
+function gcc-apt-get-install ()
+{
+  # todo: completion?
+
+  # approximate size taken up by installing:
+  # gcc:      ~423 MB
+  # g++:      ~220 MB
+  # gfortran: ~200 MB
+  #
+  sudo apt-get install \
+  "gcc-$1" \
+  "g++-$1" \
+  "gfortran-$1"
+
+  \sudo -k
+}
+
+
+function clang-apt-get-install ()
+{
+  # don't need to separately install "clang++-$1".
+  # approximate size taken up by installing: # 722 MB
+  sudo apt-get install "clang-$1"
+
+  \sudo -k
+}

development/lyx-tester/vbox_commands.txt

@@ -0,0 +1,23 @@
+# This file is useful when creating a base image, and then cloning it.
+# Although none of the commands in here is needed to run lyx-tester,
+# running some preparatory commands can save time for clones.
+
+
+# For the base image, run the following:
+sudo ./install-tl-ubuntu --only-apt-and-dpkg --hebrew
+
+# Then, for clones, just need to run the following
+# To run lyx-tester, consider one of the following commands:
+
+sudo ./lyx-tester --itl-options "--no-apt-or-dpkg"
+
+sudo ./lyx-tester --itl-options "--no-apt-or-dpkg --pretest"
+
+sudo ./lyx-tester --qt-from "git" --itl-options "--no-apt-or-dpkg"
+
+sudo ./lyx-tester --itl-options "--no-apt-or-dpkg --iso /home/$USER/Desktop/texlive2018.iso"
+
+
+# To update LyX and run the ctests, do the following:
+
+mylyxcd master && git pull && git clean -xdf && git reset --hard && GITHASH="$(git rev-parse --short HEAD)" && ~/Desktop/lyx-tester/lyxbuild && cd ../CMakeBuild/ && ctest > ~/"Desktop/ctests_$(date "+%Y-%m-%d")_${GITHASH}.log"

development/lyx-tester/vbox_prep.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# This script is helpful for preparing a fresh installation of Ubuntu
+# on a virtual box, whose only purpose is to run lyx-tester.
+
+# Usage: ./vbox_prep.sh
+# (no need to run with sudo)
+
+set -e
+set -u
+
+ubuntu_ver="$( lsb_release -sr )"
+ubuntu_ver_num="${ubuntu_ver/\./}"
+
+
+# openssh-server: allows to ssh from your computer into the virtual box. This
+#   is helpful for remotely ssh'ing into the host computer and then (thanks to
+#   this line) ssh'ing into the virtual box.
+# vim: useful to have a terminal editor when ssh'ing in
+# git: probably already installed, unless downloading this script from browser
+sudo apt-get -y install openssh-server vim git
+
+# useful e.g. for the LyX keytests. Otherwise when going to screensave the keys
+# are written to the password.
+gsettings set org.gnome.desktop.screensaver lock-enabled false
+gsettings set org.gnome.desktop.screensaver ubuntu-lock-on-suspend false
+gsettings set org.gnome.desktop.session idle-delay 0
+
+# Enable sources and install dependencies of "lyx" Ubuntu package.
+#
+# This chunk is shared by lyx-tester/lyx-tester, lyx-tester/vbox_prep.sh, master.sgk, vboxBigInstall.sh.
+# Needed for "apt-get build-dep".
+if [ ${ubuntu_ver_num} -ge 2404 ]; then
+  # Don't really need to sub the entry for security, but easy to just keep as is.
+  # use '$' so that this command can be run multiple times and will only make the sub once.
+  sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+else
+  sudo perl -pi -e 'next if /-backports/; s/^# (deb-src [^ ]+ [^ -]+ universe)$/$1/' /etc/apt/sources.list
+fi
+#
+sudo apt-get --yes update
+sudo apt-get -y build-dep lyx

src/Text.cpp

@@ -5620,10 +5620,11 @@ void Text::dispatch(Cursor & cur, FuncRequest & cmd)
 		FileName const tabstyle = libFileSearch("tabletemplates",
 							style + suffix + ".lyx", "lyx");
 		if (tabstyle.empty())
-			    break;
+			break;
 		UndoGroupHelper ugh(cur.buffer());
 		cur.recordUndo();
-		FuncRequest cmd2(LFUN_FILE_INSERT, tabstyle.absFileName() + " ignorelang");
+		FuncRequest cmd2(LFUN_FILE_INSERT,
+				 quoteName(tabstyle.absFileName()) + " ignorelang");
 		lyx::dispatch(cmd2);
 		// go into table
 		cur.backwardPos();

src/frontends/qt/GuiDocument.cpp

@@ -3644,8 +3644,6 @@ void GuiDocument::getTableStyles()
 			QString data = fn.left(fn.lastIndexOf(".lyx"));
 			QString guiname = data;
 			guiname = toqstr(translateIfPossible(qstring_to_ucs4(guiname.replace('_', ' '))));
-			QString relpath = toqstr(makeRelPath(qstring_to_ucs4(fn),
-							     qstring_to_ucs4(dir)));
 			if (textLayoutModule->tableStyleCO->findData(data) == -1)
 				textLayoutModule->tableStyleCO->addItem(guiname, data);
 		}

src/frontends/qt/GuiTabularCreate.cpp

@@ -63,8 +63,6 @@ void GuiTabularCreate::getFiles()
 			QString data = fn.left(fn.lastIndexOf(".lyx"));
 			QString guiname = data;
 			guiname = toqstr(translateIfPossible(qstring_to_ucs4(guiname.replace('_', ' '))));
-			QString relpath = toqstr(makeRelPath(qstring_to_ucs4(fn),
-							     qstring_to_ucs4(dir)));
 			if (styleCO->findData(data) == -1)
 				styleCO->addItem(guiname, data);
 		}