Merge remote-tracking branch 'upstream/maint-26' into fix-db-match-ma…

…p-copy-bug-26

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.194.0/containers/cpp
+{
+	"name": "Erlang/OTP",
+	"build": {
+	    "dockerfile": "../.github/dockerfiles/Dockerfile.ubuntu-base",
+            "context": "../.github",
+            "args": {
+                "BASE": "mcr.microsoft.com/vscode/devcontainers/base:focal",
+                "USER": "vscode",
+                "GROUP": "vscode"
+            }
+	},
+	"runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-vscode.cpptools",
+		"erlang-ls.erlang-ls"	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "gcc -v",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

.dir-locals.el

@@ -2,6 +2,7 @@
 (
  (erlang-mode (indent-tabs-mode . nil))
  (autoconf-mode (indent-tabs-mode . nil))
+ (m4-mode (indent-tabs-mode . nil))
  (java-mode (indent-tabs-mode . nil))
  (perl-mode (indent-tabs-mode . nil))
  (xml-mode (indent-tabs-mode . nil))

.github/dockerfiles/Dockerfile.32-bit

@@ -1,4 +1,5 @@
-FROM docker.pkg.github.com/erlang/otp/i386-debian-base
+ARG BASE=ghcr.io/erlang/otp/i386-debian-base
+FROM $BASE
 
 ARG MAKEFLAGS=-j4
 ENV MAKEFLAGS=$MAKEFLAGS \
@@ -16,14 +17,8 @@ ENV CFLAGS="-O2 -g -Werror"
 
 ## Configure, check that no application are disabled, then make and then build doc chunks
 RUN ./configure --with-ssl --prefix=/otp && \
-        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi
-
-RUN make && make install && \
-        make install-docs DOC_TARGETS=chunks
+        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
+    make && make install
 
 ## Disable -Werror as testcases do not compile with it on
 ENV CFLAGS="-O2 -g"
-
-RUN TESTSUITE_ROOT=/tests ./otp_build tests
-
-ENTRYPOINT ["bash","-c"]

.github/dockerfiles/Dockerfile.64-bit

@@ -1,32 +1,35 @@
-FROM docker.pkg.github.com/erlang/otp/ubuntu-base
-## We do a SSA lint check here
-ENV ERL_COMPILER_OPTIONS=ssalint
+ARG BASE=ghcr.io/erlang/otp/ubuntu-base
+FROM $BASE
 
-ARG MAKEFLAGS=-j4
+ARG MAKEFLAGS=$MAKEFLAGS
 ENV MAKEFLAGS=$MAKEFLAGS \
-        ERLC_USE_SERVER=yes \
-        ERL_TOP=/buildroot/otp \
-        PATH=/buildroot/otp/bin:$PATH
+    ERL_TOP=/buildroot/otp \
+    PATH="/Erlang ∅⊤℞/bin":/buildroot/otp/bin:$PATH
 
 ARG ARCHIVE=./otp.tar.gz
 COPY $ARCHIVE /buildroot/otp.tar.gz
 RUN cd /buildroot && tar -xzf ./otp.tar.gz
 
 WORKDIR /buildroot/otp/
 
-ENV CC=clang CXX=clang++ \
-        CFLAGS="-O2 -g -Werror"
+ENV CFLAGS="-O2 -g -Werror"
 
-## Configure, check that no application are disabled and then make
-# We need --with-ssl-lib-subdir=lib/x86_64-linux-gnu since clang does not
-# give us this information such as gcc does...
-RUN  ./configure --with-ssl --with-ssl-lib-subdir=lib/x86_64-linux-gnu --prefix=/otp && \
-        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
-        make && make install
+## Configure (if not cached), check that no application are disabled and then make
+RUN if [ ! -f Makefile ]; then \
+      touch README.md && \
+      ./configure --prefix="/Erlang ∅⊤℞" && \
+      if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
+      find . -type f -newer README.md | xargs tar --transform 's:^./:otp/:' -cf ../otp_cache.tar; \
+    fi && \
+    make && make docs DOC_TARGETS=chunks && \
+    sudo make install install-docs DOC_TARGETS=chunks
 
 ## Disable -Werror as testcases do not compile with it on
 ENV CFLAGS="-O2 -g"
 
-RUN TESTSUITE_ROOT=/tests ./otp_build tests
-
-ENTRYPOINT ["bash","-c"]
+## Update init.sh with correct env vars
+RUN echo "export MAKEFLAGS=$MAKEFLAGS" > /buildroot/env.sh && \
+    echo "export ERLC_USE_SERVER=$ERLC_USE_SERVER" >> /buildroot/env.sh && \
+    echo "export ERL_TOP=\"$ERL_TOP\"" >> /buildroot/env.sh && \
+    echo "export PATH=\"$PATH\"" >> /buildroot/env.sh && \
+    echo "export ERL_LIBS=\"$ERL_LIBS\"" >> /buildroot/env.sh

.github/dockerfiles/Dockerfile.clang

@@ -0,0 +1,32 @@
+ARG BASE=ghcr.io/erlang/otp/ubuntu-base
+FROM $BASE
+## We do a SSA lint check here
+ENV ERL_COMPILER_OPTIONS=ssalint
+
+ARG MAKEFLAGS=-j4
+ENV MAKEFLAGS=$MAKEFLAGS \
+    ERLC_USE_SERVER=yes \
+    ERL_TOP=/buildroot/otp \
+    PATH=/otp/bin:/buildroot/otp/bin:$PATH
+
+RUN sudo apt-get update && sudo apt-get install -y clang
+
+ARG ARCHIVE=./otp.tar.gz
+COPY $ARCHIVE /buildroot/otp.tar.gz
+RUN cd /buildroot && tar -xzf ./otp.tar.gz
+
+WORKDIR /buildroot/otp/
+
+ENV CC=clang CXX=clang++ \
+        CFLAGS="-O2 -g -Werror"
+
+## Configure, check that no application are disabled and then make
+# We need --with-ssl-lib-subdir=lib/x86_64-linux-gnu since clang does not
+# give us this information such as gcc does...
+RUN  ./configure --with-ssl --with-ssl-lib-subdir=lib/x86_64-linux-gnu --prefix=/otp && \
+        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
+    make && sudo make install && \
+    make FLAVOR=emu && sudo make install FLAVOR=emu
+
+## Disable -Werror as testcases do not compile with it on
+ENV CFLAGS="-O2 -g"

.github/dockerfiles/Dockerfile.cross-compile

@@ -1,7 +1,8 @@
 ##
 ## This docker file will build Erlang on 32-bit to 64-bit x86
 ##
-FROM docker.pkg.github.com/erlang/otp/i386-debian-base as build
+ARG BASE=ghcr.io/erlang/otp/i386-debian-base
+FROM $BASE as build
 
 ARG MAKEFLAGS=-j4
 ENV MAKEFLAGS=$MAKEFLAGS \
@@ -17,13 +18,6 @@ RUN tar xzf ../otp.tar.gz
 ## Build the bootstrap system
 RUN cd $ERL_TOP && CFLAGS="-Wall -O2 -g" ./configure && make && make install
 
-## Build pre-build tar ball
-RUN cd $ERL_TOP && scripts/build-otp-tar -o /buildroot/otp_clean_src.tar.gz /buildroot/otp_src.tar.gz \
-        -b /buildroot/bootstrap/otp/ /buildroot/otp.tar.gz
-
-## Prepare for a new build using pre-built tar ball
-RUN cd ../ && tar -xzf ./otp_src.tar.gz
-
 ENV HOST=$HOST_TRIP \
         CC=$HOST_TRIP-gcc \
         CPPFLAGS="--sysroot=/buildroot/sysroot" \
@@ -39,15 +33,16 @@ ENV HOST=$HOST_TRIP \
         erl_xcomp_sysroot=/buildroot/sysroot \
         ERL_TOP=/buildroot/otp
 
+RUN cd /buildroot && tar xzf otp.tar.gz
+
 WORKDIR /buildroot/otp
 
 ## Build the cross system
 # We cannot use config.guess for --build since its value clashes with the
 # canonical value of host...
-RUN ./configure --prefix=/otp/ --host=$HOST --build=x86-pc-linux-gnu
-RUN OTP_SMALL_BUILD=true V=1 make
-
-RUN make odbc && make install
+RUN ./configure --prefix=/otp/ --host=$HOST --build=x86-pc-linux-gnu && \
+    OTP_SMALL_BUILD=true V=1 make && \
+    make install
 
 ## Build the cross tests
 ENV CFLAGS="--sysroot=/buildroot/sysroot -O2 -g"
@@ -64,5 +59,3 @@ COPY --from=build /otp /otp
 COPY --from=build /buildroot/otp/release/tests /tests
 
 ENV PATH=/otp/bin:$PATH
-
-ENTRYPOINT ["bash","-c"]

.github/dockerfiles/Dockerfile.ubuntu-base

@@ -1,16 +1,122 @@
 ##
 ## This docker file will build a base image for building Erlang/OTP
 ##
-ARG BASE=ubuntu:20.04
+ARG BASE=gitpod/workspace-full
 FROM $BASE
 
-ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssl-dev unixodbc-dev libgmp3-dev libwxbase3.0-dev libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev libsctp-dev lksctp-tools"
+ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssl-dev unixodbc-dev libsctp-dev lksctp-tools libgmp3-dev libwxbase3.0-dev libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev"
+
+USER root
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+## Install build tools
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get install -y build-essential m4 autoconf fop xsltproc \
+    default-jdk libxml2-utils flex pkg-config locales tzdata sudo ${INSTALL_LIBS} && \
+    sed -i 's@# en_US.UTF-8@en_US.UTF-8@g' /etc/locale.gen && locale-gen && \
+    update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0
+
+ARG MAKEFLAGS=-j4
+ENV MAKEFLAGS=$MAKEFLAGS \
+    ERLC_USE_SERVER=yes
+
+ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+
+ARG USER=gitpod
+ARG GROUP=gitpod
+ARG gid=10
+ARG uid=421
+
+RUN echo "Europe/Stockholm" > /etc/timezone && \
+    ln -snf /usr/share/zoneinfo/$(cat /etc/timezone) /etc/localtime && \
+    if ! grep ":${gid}:$" /etc/group; then groupadd -g ${gid} localgroup; fi && \
+    if [ ! -d /home/${USER} ]; then useradd -rm -d /home/${USER} -s /bin/sh -g ${gid} -G ${gid},sudo -u ${uid} ${USER}; fi && \
+    echo "${USER} ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \
+    echo "/buildroot/** r," >> /etc/apparmor.d/local/usr.sbin.named && \
+    echo "/tests/** r," >> /etc/apparmor.d/local/usr.sbin.named
+
+## Java and log4j are used by fop to build documentation
+COPY --chown=${USER}:${GROUP} dockerfiles/log4j.properties /home/${USER}/
+ENV JAVA_ARGS="-Dlog4j.configuration=file://home/${USER}/log4j.properties"
+
+ENV OTP_STRICT_INSTALL=yes
+
+RUN mkdir /buildroot /tests /otp && chown ${USER}:${GROUP} /buildroot /tests /otp
+
+## We install the latest version of the previous three releases in order to do
+## backwards compatability testing of Erlang.
+RUN apt-get install -y git curl && \
+    curl -L https://raw.githubusercontent.com/kerl/kerl/master/kerl > /usr/bin/kerl && \
+    chmod +x /usr/bin/kerl && \
+    kerl update releases && \
+    LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \
+    for release in $(seq $(( LATEST - 2 )) $(( LATEST ))); do \
+      VSN=$(kerl list releases | grep "^$release" | tail -1); \
+      if [ $release = $LATEST ]; then \
+        echo "/usr/local/lib/erlang-${VSN}/bin" > /home/${USER}/LATEST; \
+      fi && \
+      kerl build ${VSN} ${VSN} && \
+      kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \
+    done && \
+    rm -rf ~/.kerl
+
+## Install test tools
+## EXTRA_LIBS are installed using a for loop because of bugs in the erlang-doc deb package
+## Apache2 may already be installed, if so we do not want to install it again
+ARG EXTRA_LIBS="erlang erlang-doc"
+RUN apt-get install -y \
+    unixodbc odbc-postgresql postgresql ssh openssh-server groff-base gdb \
+    tinyproxy bind9 nsd expect vsftpd python emacs nano vim \
+    linux-tools-common linux-tools-generic jq \
+    xvfb libgl1-mesa-dri && \
+    for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \
+    if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi
+RUN apt-get install -y linux-tools-$(uname -r) || true
+
+## We use tmux to test terminals
+RUN apt-get install -y libevent-dev libutf8proc-dev && \
+    cd /tmp && wget https://github.com/tmux/tmux/releases/download/3.2a/tmux-3.2a.tar.gz && \
+    tar xvzf tmux-3.2a.tar.gz && cd tmux-3.2a && \
+    ./configure --enable-static --enable-utf8proc && \
+    make && make install
+
+## Setup progres so that the odbc test can run
+USER postgres
+
+RUN service postgresql start && \
+    psql -c "CREATE USER odbctest WITH SUPERUSER PASSWORD 'odbctest';" && \
+    createdb -O odbctest odbctest && \
+    service postgresql stop
+
+COPY --chown=root:${GROUP} dockerfiles/odbc.ini /etc/
+COPY --chown=root:${GROUP} dockerfiles/odbcinst.ini /etc/
+
+USER ${USER}
+
+## Need to set USER and create a keygen file for ssh tests to pass
+ENV USER=${USER}
+RUN ssh-keygen -q -t rsa -N '' -f $HOME/.ssh/id_rsa && \
+    cp $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys
+
+COPY --chown=${USER}:${GROUP} dockerfiles/init.sh /buildroot/
+
+WORKDIR /buildroot/
+
+## Install test tools rebar3, proper and jsx
+RUN export PATH="$(cat /home/${USER}/LATEST):${PATH}" && \
+    latest () { \
+    local VSN=$(curl -sL "https://api.github.com/repos/$1/tags" | jq -r ".[] | .name" | grep -E '^v?[0-9]' | sort -V | tail -1); \
+    curl -sL "https://github.com/$1/archive/$VSN.tar.gz" > $(basename $1).tar.gz; \
+    } && \
+    latest erlang/rebar3 && ls -la && \
+    (tar xzf rebar3.tar.gz && cd rebar3-* && ./bootstrap && sudo cp rebar3 /usr/bin) && \
+    latest proper-testing/proper && \
+    (tar xzf proper.tar.gz && mv proper-* proper && cd proper && make) && \
+    latest talentdeficit/jsx && \
+    (tar xzf jsx.tar.gz && mv jsx-* jsx && cd jsx && rebar3 compile)
 
-RUN apt-get update && \
-        apt-get -y upgrade && \
-        apt-get install -y build-essential m4 autoconf fop xsltproc clang clang-format \
-        default-jdk libxml2-utils $INSTALL_LIBS
+ENV ERL_LIBS=/buildroot/proper:/buildroot/jsx
 
-RUN update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0
+ENTRYPOINT ["/buildroot/init.sh"]

.github/dockerfiles/init.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+if [ -f "/buildroot/env.sh" ]; then
+    . "/buildroot/env.sh"
+fi
+
+sudo mkdir -p -m0755 /var/run/sshd
+
+sudo /usr/sbin/sshd
+
+sudo service postgresql start
+
+sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic"
+sudo -E bash -c "apt-get install -y linux-tools-$(uname-r)" || true
+
+sudo bash -c "Xvfb :99 -ac -screen 0 1920x1080x24 -nolisten tcp" &
+export DISPLAY=:99
+
+PATH="$PATH:$(ls -1d /usr/local/lib/erlang-*/bin | tr '\n' ':')"
+
+exec /bin/bash -c "$1"

.github/dockerfiles/log4j.properties

@@ -0,0 +1,9 @@
+log4j.rootLogger = INFO, console
+
+log4j.appender.console = org.apache.log4j.ConsoleAppender
+
+log4j.appender.console.layout = org.apache.log4j.PatternLayour
+log4j.appender.console.layout.ConversionPattern = %d [%t] %-5p %c{1} - %m%n
+
+log4j.logger.org.apache.fop = INFO
+log4j.logger.org.apache.fop.render = INFO