CI: test_environment: sync env install scripts

Tools/environment_install/install-ROS-ubuntu.sh

@@ -4,6 +4,7 @@ set -e
 # set -x
 
 ROS_WS_ROOT=$HOME/ardupilot-ws
+AP_GZ_ROOT=$HOME/ardupilot_gz_ws
 
 red=`tput setaf 1`
 green=`tput setaf 2`
@@ -230,6 +231,7 @@ if maybe_prompt_user "Add ardupilot-ws to your home folder [N/y]?" ; then
     if [ ! -d $ROS_WS_ROOT ]; then
         mkdir -p $ROS_WS_ROOT/src
         pushd $ROS_WS_ROOT
+        source /opt/ros/${ROS_VERSION}/setup.bash
         catkin init
         pushd src
         git clone https://github.com/ArduPilot/ardupilot_ros.git
@@ -246,6 +248,30 @@ else
     echo "Skipping adding ardupilot_ws to your home folder."
 fi
 
+
+if maybe_prompt_user "Add ardupilot_gazebo to your home folder [N/y]?" ; then
+    if [ ! -d $AP_GZ_ROOT ]; then
+        sudo apt install gz-garden rapidjson-dev
+        mkdir -p $AP_GZ_ROOT/src
+        pushd $AP_GZ_ROOT/src
+        git clone https://github.com/ArduPilot/ardupilot_gazebo
+        pushd ardupilot_gazebo
+        mkdir build && pushd build
+        cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo
+        make -j4
+        popd
+        popd
+        popd
+        echo "export GZ_SIM_SYSTEM_PLUGIN_PATH=${AP_GZ_ROOT}/src/ardupilot_gazebo/build:${GZ_SIM_SYSTEM_PLUGIN_PATH}" >> ~/.bashrc
+        echo "export GZ_SIM_RESOURCE_PATH=${AP_GZ_ROOT}/src/ardupilot_gazebo/models:${AP_GZ_ROOT}/src/ardupilot_gazebo/worlds:${GZ_SIM_RESOURCE_PATH}" >> ~/.bashrc
+    else
+        heading "${red}ardupilot_gazebo already exists, skipping...${reset}"
+    fi
+
+else
+    echo "Skipping adding ardupilot_gazebo to your home folder."
+fi
+
 heading "${green}Adding setup.bash, ROS_MASTER_URI and ROS_HOSTNAME to .bashrc ${reset}"
 grep -q -F "source $ROS_WS_ROOT/devel/setup.bash" ~/.bashrc || echo "source $ROS_WS_ROOT/devel/setup.bash" >> ~/.bashrc
 source ~/.bashrc

Tools/environment_install/install-prereqs-arch.sh

@@ -50,10 +50,25 @@ function maybe_prompt_user() {
     fi
 }
 
-sudo usermod -a -G uucp $USER
+sudo usermod -a -G uucp "$USER"
 
-sudo pacman -Sy --noconfirm --needed $BASE_PKGS $SITL_PKGS $PX4_PKGS
-pip3 -q install --user -U $PYTHON_PKGS
+sudo pacman -Syu --noconfirm --needed $BASE_PKGS $SITL_PKGS $PX4_PKGS
+
+python3 -m venv "$HOME"/venv-ardupilot
+
+# activate it:
+SOURCE_LINE="source $HOME/venv-ardupilot/bin/activate"
+$SOURCE_LINE
+
+if [[ -z "${DO_PYTHON_VENV_ENV}" ]] && maybe_prompt_user "Make ArduPilot venv default for python [N/y]?" ; then
+    DO_PYTHON_VENV_ENV=1
+fi
+
+if [[ $DO_PYTHON_VENV_ENV -eq 1 ]]; then
+    echo "$SOURCE_LINE" >> ~/.bashrc
+fi
+
+pip3 -q install -U $PYTHON_PKGS
 
 (
     cd /usr/lib/ccache
@@ -78,7 +93,7 @@ exportline="export PATH=$OPT/$ARM_ROOT/bin:\$PATH";
 if ! grep -Fxq "$exportline" ~/.bashrc ; then
     if maybe_prompt_user "Add $OPT/$ARM_ROOT/bin to your PATH [N/y]?" ; then
         echo "$exportline" >> ~/.bashrc
-        . ~/.bashrc
+        . "$HOME/.bashrc"
     else
         echo "Skipping adding $OPT/$ARM_ROOT/bin to PATH."
     fi
@@ -88,15 +103,15 @@ exportline2="export PATH=$CWD/$ARDUPILOT_TOOLS:\$PATH";
 if  ! grep -Fxq "$exportline2" ~/.bashrc ; then
     if maybe_prompt_user "Add $CWD/$ARDUPILOT_TOOLS to your PATH [N/y]?" ; then
         echo "$exportline2" >> ~/.bashrc
-        . ~/.bashrc
+        . "$HOME/.bashrc"
     else
         echo "Skipping adding $CWD/$ARDUPILOT_TOOLS to PATH."
     fi
 fi
 
 SCRIPT_DIR=$(dirname $(realpath ${BASH_SOURCE[0]}))
 (
-    cd $SCRIPT_DIR
+    cd "$SCRIPT_DIR"
     git submodule update --init --recursive
 )
 

Tools/environment_install/install-prereqs-mac.sh

@@ -118,7 +118,7 @@ if maybe_prompt_user "Install python using pyenv [N/y]?" ; then
 
         pushd $HOME/.pyenv
         git fetch --tags
-        git checkout v2.3.9
+        git checkout v2.3.12
         popd
         exportline="export PYENV_ROOT=\$HOME/.pyenv"
         echo $exportline >> ~/$SHELL_LOGIN

Tools/environment_install/install-prereqs-ubuntu.sh

@@ -59,20 +59,37 @@ fi
 
 # Checking Ubuntu release to adapt software version to install
 RELEASE_CODENAME=$(lsb_release -c -s)
+
+# translate Mint-codenames to Ubuntu-codenames based on https://www.linuxmint.com/download_all.php
+case ${RELEASE_CODENAME} in
+    vanessa)
+        RELEASE_CODENAME='jammy'
+        ;;
+    una | uma | ulyssa | ulyana | jolnir)
+        RELEASE_CODENAME='focal'
+        ;;
+    tricia | tina | tessa | tara)
+        RELEASE_CODENAME='bionic'
+        ;;
+    elsie)
+        RELEASE_CODENAME='bullseye'
+        ;;
+esac
+
 PYTHON_V="python3"  # starting from ubuntu 20.04, python isn't symlink to default python interpreter
 PIP=pip3
 
 if [ ${RELEASE_CODENAME} == 'bionic' ] ; then
     SITLFML_VERSION="2.4"
     SITLCFML_VERSION="2.4"
-    PYTHON_V="python"
-    PIP=pip2
+    PYTHON_V="python3"
+    PIP=pip3
 elif [ ${RELEASE_CODENAME} == 'buster' ]; then
     SITLFML_VERSION="2.5"
     SITLCFML_VERSION="2.5"
-    PYTHON_V="python"
-    PIP=pip2
-elif [ ${RELEASE_CODENAME} == 'focal' ] || [ ${RELEASE_CODENAME} == 'ulyssa' ]; then
+    PYTHON_V="python3"
+    PIP=pip3
+elif [ ${RELEASE_CODENAME} == 'focal' ]; then
     SITLFML_VERSION="2.5"
     SITLCFML_VERSION="2.5"
     PYTHON_V="python3"
@@ -82,6 +99,16 @@ elif [ ${RELEASE_CODENAME} == 'jammy' ]; then
     SITLCFML_VERSION="2.5"
     PYTHON_V="python3"
     PIP=pip3
+elif [ ${RELEASE_CODENAME} == 'lunar' ]; then
+    SITLFML_VERSION="2.5"
+    SITLCFML_VERSION="2.5"
+    PYTHON_V="python3"
+    PIP=pip3
+elif [ ${RELEASE_CODENAME} == 'mantic' ]; then
+    SITLFML_VERSION="2.5"
+    SITLCFML_VERSION="2.5"
+    PYTHON_V="python3"
+    PIP=pip3
 elif [ ${RELEASE_CODENAME} == 'groovy' ] ||
          [ ${RELEASE_CODENAME} == 'bullseye' ]; then
     SITLFML_VERSION="2.5"
@@ -124,28 +151,35 @@ else
 fi
 
 # Lists of packages to install
-BASE_PKGS="build-essential ccache g++ gawk git make wget"
-if [ ${RELEASE_CODENAME} == 'bionic' ]; then
-    # use fixed version for package that drop python2 support
-    PYTHON_PKGS="future lxml pymavlink MAVProxy pexpect flake8==3.7.9 requests==2.27.1 monotonic==1.6 geocoder empy configparser==4.0.2 click==7.1.2 decorator==4.4.2 dronecan"
-else
-    PYTHON_PKGS="future lxml pymavlink MAVProxy pexpect flake8 geocoder empy dronecan"
-fi
+BASE_PKGS="build-essential ccache g++ gawk git make wget valgrind screen"
+PYTHON_PKGS="future lxml pymavlink pyserial MAVProxy pexpect geocoder empy ptyprocess dronecan"
+PYTHON_PKGS="$PYTHON_PKGS flake8"
 
 # add some Python packages required for commonly-used MAVProxy modules and hex file generation:
 if [[ $SKIP_AP_EXT_ENV -ne 1 ]]; then
-    if [ ${RELEASE_CODENAME} == 'bionic' ]; then
-        PYTHON_PKGS="$PYTHON_PKGS pygame==2.0.3 intelhex"
-    else
-        PYTHON_PKGS="$PYTHON_PKGS pygame intelhex"
-    fi
+    PYTHON_PKGS="$PYTHON_PKGS pygame intelhex"
 fi
 ARM_LINUX_PKGS="g++-arm-linux-gnueabihf $INSTALL_PKG_CONFIG"
 # python-wxgtk packages are added to SITL_PKGS below
+
+if [ ${RELEASE_CODENAME} == 'lunar' ] ||
+   [ ${RELEASE_CODENAME} == 'mantic' ]; then
+    # on Lunar (and presumably later releases), we install in venv, below
+    PYTHON_PKGS+=" numpy pyparsing psutil"
+    SITL_PKGS="python3-dev"
+else
 SITL_PKGS="libtool libxml2-dev libxslt1-dev ${PYTHON_V}-dev ${PYTHON_V}-pip ${PYTHON_V}-setuptools ${PYTHON_V}-numpy ${PYTHON_V}-pyparsing ${PYTHON_V}-psutil"
+fi
+
 # add some packages required for commonly-used MAVProxy modules:
 if [[ $SKIP_AP_GRAPHIC_ENV -ne 1 ]]; then
+    if [ ${RELEASE_CODENAME} == 'lunar' ] ||
+       [ ${RELEASE_CODENAME} == 'mantic' ]; then
+        PYTHON_PKGS+=" matplotlib scipy opencv-python pyyaml"
+        SITL_PKGS+=" xterm libcsfml-dev libcsfml-audio${SITLCFML_VERSION} libcsfml-dev libcsfml-graphics${SITLCFML_VERSION} libcsfml-network${SITLCFML_VERSION} libcsfml-system${SITLCFML_VERSION} libcsfml-window${SITLCFML_VERSION} libsfml-audio${SITLFML_VERSION} libsfml-dev libsfml-graphics${SITLFML_VERSION} libsfml-network${SITLFML_VERSION} libsfml-system${SITLFML_VERSION} libsfml-window${SITLFML_VERSION}"
+  else
   SITL_PKGS="$SITL_PKGS xterm ${PYTHON_V}-matplotlib ${PYTHON_V}-serial ${PYTHON_V}-scipy ${PYTHON_V}-opencv libcsfml-dev libcsfml-audio${SITLCFML_VERSION} libcsfml-dev libcsfml-graphics${SITLCFML_VERSION} libcsfml-network${SITLCFML_VERSION} libcsfml-system${SITLCFML_VERSION} libcsfml-window${SITLCFML_VERSION} libsfml-audio${SITLFML_VERSION} libsfml-dev libsfml-graphics${SITLFML_VERSION} libsfml-network${SITLFML_VERSION} libsfml-system${SITLFML_VERSION} libsfml-window${SITLFML_VERSION} ${PYTHON_V}-yaml"
+  fi
 fi
 if [[ $SKIP_AP_COV_ENV -ne 1 ]]; then
   # Coverage utilities
@@ -218,8 +252,7 @@ sudo usermod -a -G dialout $USER
 echo "Done!"
 
 # Add back python symlink to python interpreter on Ubuntu >= 20.04
-if [ ${RELEASE_CODENAME} == 'focal' ] ||
-       [ ${RELEASE_CODENAME} == 'ulyssa' ];
+if [ ${RELEASE_CODENAME} == 'focal' ];
 then
     BASE_PKGS+=" python-is-python3"
     SITL_PKGS+=" libpython3-stdlib" # for argparse
@@ -228,18 +261,27 @@ elif [ ${RELEASE_CODENAME} == 'groovy' ] ||
          [ ${RELEASE_CODENAME} == 'jammy' ]; then
     BASE_PKGS+=" python-is-python3"
     SITL_PKGS+=" libpython3-stdlib" # for argparse
-else
+elif [ ${RELEASE_CODENAME} == 'lunar' ]; then
+    SITL_PKGS+=" libpython3-stdlib" # for argparse
+elif [ ${RELEASE_CODENAME} == 'buster' ]; then
+    SITL_PKGS+=" libpython3-stdlib" # for argparse
+elif [ ${RELEASE_CODENAME} != 'mantic' ]; then
   SITL_PKGS+=" python-argparse"
 fi
 
 # Check for graphical package for MAVProxy
 if [[ $SKIP_AP_GRAPHIC_ENV -ne 1 ]]; then
-  if [ ${RELEASE_CODENAME} == 'bullseye' ]; then
+  if [ ${RELEASE_CODENAME} == 'bullseye' ] ||
+         [ ${RELEASE_CODENAME} == 'buster' ]; then
     SITL_PKGS+=" libjpeg62-turbo-dev"
   elif [ ${RELEASE_CODENAME} == 'groovy' ] ||
-           [ ${RELEASE_CODENAME} == 'focal' ] ||
-           [ ${RELEASE_CODENAME} == 'ulyssa'  ]; then
+           [ ${RELEASE_CODENAME} == 'focal' ]; then
     SITL_PKGS+=" libjpeg8-dev"
+  elif [ ${RELEASE_CODENAME} == 'lunar' ]; then
+    SITL_PKGS+=" libgtk-3-dev libwxgtk3.2-dev "
+  elif [ ${RELEASE_CODENAME} == 'mantic' ]; then
+    SITL_PKGS+=" libgtk-3-dev libwxgtk3.2-dev "
+    # see below
   elif apt-cache search python-wxgtk3.0 | grep wx; then
       SITL_PKGS+=" python-wxgtk3.0"
   elif apt-cache search python3-wxgtk4.0 | grep wx; then
@@ -250,10 +292,19 @@ if [[ $SKIP_AP_GRAPHIC_ENV -ne 1 ]]; then
       SITL_PKGS+=" python-wxgtk2.8"
       SITL_PKGS+=" fonts-freefont-ttf libfreetype6-dev libjpeg8-dev libpng12-0 libportmidi-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsdl1.2-dev"  # for pygame
   fi
-  if [ ${RELEASE_CODENAME} == 'bullseye' ] ||
+
+  if [ ${RELEASE_CODENAME} == 'lunar' ]; then
+      PYTHON_PKGS+=" opencv-python"
+      SITL_PKGS+=" python3-wxgtk4.0"
+      SITL_PKGS+=" fonts-freefont-ttf libfreetype6-dev libpng16-16 libportmidi-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsdl1.2-dev"  # for pygame
+  elif [ ${RELEASE_CODENAME} == 'mantic' ]; then
+      PYTHON_PKGS+=" wxpython opencv-python"
+      SITL_PKGS+=" python3-wxgtk4.0"
+      SITL_PKGS+=" fonts-freefont-ttf libfreetype6-dev libpng16-16 libportmidi-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsdl1.2-dev"  # for pygame
+  elif [ ${RELEASE_CODENAME} == 'bullseye' ] ||
          [ ${RELEASE_CODENAME} == 'groovy' ] ||
+         [ ${RELEASE_CODENAME} == 'buster' ] ||
          [ ${RELEASE_CODENAME} == 'focal' ] ||
-         [ ${RELEASE_CODENAME} == 'ulyssa' ] ||
          [ ${RELEASE_CODENAME} == 'jammy' ]; then
     SITL_PKGS+=" python3-wxgtk4.0"
     SITL_PKGS+=" fonts-freefont-ttf libfreetype6-dev libpng16-16 libportmidi-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsdl1.2-dev"  # for pygame
@@ -274,12 +325,57 @@ fi
 
 # Install all packages
 $APT_GET install $BASE_PKGS $SITL_PKGS $PX4_PKGS $ARM_LINUX_PKGS $COVERAGE_PKGS
-# Update Pip and Setuptools on old distro
-if [ ${RELEASE_CODENAME} == 'bionic' ]; then
-    # use fixed version for package that drop python2 support
-    $PIP install --user -U pip==20.3 setuptools==44.0.0
+
+heading "Check if we are inside docker environment..."
+IS_DOCKER=false
+if [[ -f /.dockerenv ]] || grep -Eq '(lxc|docker)' /proc/1/cgroup ; then
+    IS_DOCKER=true
+fi
+echo "Done!"
+
+SHELL_LOGIN=".profile"
+if $IS_DOCKER; then
+    echo "Inside docker, we add the tools path into .bashrc directly"
+    SHELL_LOGIN=".ardupilot_env"
+    echo "# ArduPilot env file. Need to be loaded by your Shell." > ~/$SHELL_LOGIN
+fi
+
+PIP_USER_ARGUMENT="--user"
+
+# create a Python venv on more recent releases:
+if [ ${RELEASE_CODENAME} == 'lunar' ] ||
+   [ ${RELEASE_CODENAME} == 'mantic' ]; then
+    $APT_GET install python3.11-venv
+    python3 -m venv $HOME/venv-ardupilot
+
+    # activate it:
+    SOURCE_LINE="source $HOME/venv-ardupilot/bin/activate"
+    $SOURCE_LINE
+    PIP_USER_ARGUMENT=""
+
+    if [[ -z "${DO_PYTHON_VENV_ENV}" ]] && maybe_prompt_user "Make ArduPilot venv default for python [N/y]?" ; then
+        DO_PYTHON_VENV_ENV=1
+    fi
+
+    if [[ $DO_PYTHON_VENV_ENV -eq 1 ]]; then
+        echo $SOURCE_LINE >> ~/$SHELL_LOGIN
+    fi
+fi
+
+# try update setuptools and wheel before installing pip package that may need compilation
+$PIP install $PIP_USER_ARGUMENT -U pip setuptools wheel
+
+if [ "$GITHUB_ACTIONS" == "true" ]; then
+    PIP_USER_ARGUMENT+=" --progress-bar off"
+fi
+
+if [ ${RELEASE_CODENAME} == 'lunar' ] ||
+   [ ${RELEASE_CODENAME} == 'mantic' ]; then
+    # must do this ahead of wxPython pip3 run :-/
+    $PIP install $PIP_USER_ARGUMENT -U attrdict3
 fi
-$PIP install --user -U $PYTHON_PKGS
+
+$PIP install $PIP_USER_ARGUMENT -U $PYTHON_PKGS
 
 if [[ -z "${DO_AP_STM_ENV}" ]] && maybe_prompt_user "Install ArduPilot STM32 toolchain [N/y]?" ; then
     DO_AP_STM_ENV=1
@@ -299,20 +395,6 @@ if [[ $DO_AP_STM_ENV -eq 1 ]]; then
   install_arm_none_eabi_toolchain
 fi
 
-heading "Check if we are inside docker environment..."
-IS_DOCKER=false
-if [[ -f /.dockerenv ]] || grep -Eq '(lxc|docker)' /proc/1/cgroup ; then
-    IS_DOCKER=true
-fi
-echo "Done!"
-
-SHELL_LOGIN=".profile"
-if $IS_DOCKER; then
-    echo "Inside docker, we add the tools path into .bashrc directly"
-    SHELL_LOGIN=".ardupilot_env"
-    echo "# ArduPilot env file. Need to be loaded by your Shell." > ~/$SHELL_LOGIN
-fi
-
 heading "Adding ArduPilot Tools to environment"
 
 SCRIPT_DIR=$(dirname $(realpath ${BASH_SOURCE[0]}))

Tools/environment_install/install-prereqs-windows-andAPMSource.ps1

@@ -19,7 +19,7 @@ Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -
 Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'ln -sf /usr/bin/pip3.7 /usr/bin/pip'"
 
 Write-Output "Downloading extra Python packages (5/8)"
-Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'pip install empy pyserial pymavlink intelhex dronecan'"
+Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'pip install empy pyserial pymavlink intelhex dronecan pexpect'"
 
 Write-Output "Downloading APM source (6/8)"
 Copy-Item "APM_install.sh" -Destination "C:\cygwin64\home"

Tools/environment_install/install-prereqs-windows.ps1

@@ -19,7 +19,7 @@ Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -
 Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'ln -sf /usr/bin/pip3.7 /usr/bin/pip'"
 
 Write-Output "Downloading extra Python packages (5/7)"
-Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'pip install empy pyserial pymavlink intelhex dronecan'"
+Start-Process -wait -FilePath "C:\cygwin64\bin\bash" -ArgumentList "--login -i -c 'pip install empy pyserial pymavlink intelhex dronecan pexpect'"
 
 Write-Output "Installing ARM GCC Compiler 10-2020-Q4-Major (6/7)"
 & $PSScriptRoot\gcc-arm-none-eabi-10-2020-q4-major-win32.exe /S /P /R