From 7e3dbdc9f484332344cbbaa8dcd10c97f4808a40 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 14:01:29 +0200 Subject: [PATCH 01/16] Use Ubuntu 24.04 (WIP) --- Dockerfile | 17 ++++++++++------- install_scripts/install_r.sh | 6 ++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 092061d..c2a6baa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM ubuntu:jammy@sha256:58b87898e82351c6cf9cf5b9f3c20257bb9e2dcf33af051e12ce532d7f94e3fe +FROM ubuntu:noble@sha256:b359f1067efa76f37863778f7b6d0e8d911e3ee8efa807ad01fbf5dc1ef9006b + SHELL ["/bin/bash", "-c"] @@ -10,10 +11,12 @@ ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG DEBIAN_FRONTEND=noninteractive -# Add non root user -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd -rm -d /home/$USERNAME -s /bin/bash -g root --uid $USER_UID --gid $USER_GID $USERNAME \ - && addgroup $USERNAME staff +# Rename ubuntu user to $USERNAME +RUN usermod -l $USERNAME ubuntu \ + # Add new group called $USERNAME + && groupadd $USERNAME \ + # Add new user $USERNAME to the groups $USERNAME and staff + && usermod -a -G staff,$USERNAME $USERNAME # Create folders to mount extensions RUN mkdir -p /home/$USERNAME/.vscode-server/extensions \ @@ -116,7 +119,7 @@ RUN apt-get update &&\ python3-dev \ python3-venv && \ # Python packages - pip3 install -U --no-cache-dir \ + pip3 install -U --no-cache-dir --break-system-packages \ $(grep -o '^[^#]*' package_lists/python_packages.txt | tr '\n' ' ') \ && apt-get autoclean -y \ && rm -rf /var/lib/apt/lists/* @@ -145,7 +148,7 @@ ENV PATH="/usr/local/texlive/bin/x86_64-linux:${PATH}" ENV R_VERSION=4.4.1 # Set RSPM snapshot see: -# https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2023-10-04&distribution=ubuntu-22.04 +# https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/jammy/2024-10-01 COPY install_scripts/install_r.sh /tmp/install_r.sh diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index 6f018ed..cf0a167 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -1,9 +1,7 @@ #!/bin/bash -echo "deb http://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" >> /etc/apt/sources.list -gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 -gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - -apt-get update +echo "deb http://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list +wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc apt-get update From cbb115b4fb663619540c5532c92c91d51f8fb4c1 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 14:11:12 +0200 Subject: [PATCH 02/16] Use "ubuntu" username instead of vscode --- Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2a6baa..318aec6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,17 +6,13 @@ SHELL ["/bin/bash", "-c"] ENV DISPLAY=:0 \ TZ=Europe/Berlin -ARG USERNAME=vscode +ARG USERNAME=ubuntu ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG DEBIAN_FRONTEND=noninteractive -# Rename ubuntu user to $USERNAME -RUN usermod -l $USERNAME ubuntu \ - # Add new group called $USERNAME - && groupadd $USERNAME \ - # Add new user $USERNAME to the groups $USERNAME and staff - && usermod -a -G staff,$USERNAME $USERNAME +# Add ubuntu user to ubuntu and staff groups +RUN usermod -a -G staff,$USERNAME $USERNAME # Create folders to mount extensions RUN mkdir -p /home/$USERNAME/.vscode-server/extensions \ From 698ad86807ec7567ecf6d0989c871261ae82d285 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 14:46:20 +0200 Subject: [PATCH 03/16] Remove sudo from install_r.sh --- install_scripts/install_r.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index cf0a167..699ff08 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "deb http://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list -wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc apt-get update From aa0b130b478490a6aac84338022285e03805c757 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 15:17:46 +0200 Subject: [PATCH 04/16] Use apt-key again --- install_scripts/install_r.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index 699ff08..2dc8fb5 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -1,7 +1,9 @@ #!/bin/bash echo "deb http://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list -wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 +gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - +apt-get update apt-get update From dc7be38ee387f76686cea656aac2a09bf700d3fc Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 18:00:35 +0200 Subject: [PATCH 05/16] Update install_r.sh + Add cargo to Dockerfile --- Dockerfile | 11 +++++++---- install_scripts/install_r.sh | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 318aec6..fa0f614 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele ccache \ gfortran \ netbase \ + cargo \ zip \ unzip \ xclip \ @@ -121,7 +122,7 @@ RUN apt-get update &&\ && rm -rf /var/lib/apt/lists/* # Set PATH for user installed python packages -ENV PATH="/home/vscode/.local/bin:${PATH}" +ENV PATH="/home/${USERNAME}/.local/bin:${PATH}" # Install Latex COPY install_scripts/install_latex.sh /tmp/install_latex.sh @@ -135,17 +136,19 @@ RUN chmod +x /tmp/install_latex.sh &&\ && tlmgr option -- srcfiles 0 \ && tlmgr install \ $(grep -o '^[^#]*' /tmp/latex_packages.txt | tr '\n' ' ') \ - && chown --recursive $USERNAME:$USERNAME /usr/local/texlive + && chown --recursive $USERNAME:$USERNAME /usr/local/texlive \ + && cargo install tex-fmt -# Set Latex Path +# Set Latex Paths ENV PATH="/usr/local/texlive/bin/x86_64-linux:${PATH}" +ENV PATH="/home/${USERNAME}/.cargo/bin:${PATH}" # Install R ENV R_VERSION=4.4.1 # Set RSPM snapshot see: # https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 -ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/jammy/2024-10-01 +ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/noble/2024-10-01 COPY install_scripts/install_r.sh /tmp/install_r.sh COPY package_lists/r_packages.txt /tmp/r_packages.txt diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index 2dc8fb5..b701c77 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -1,9 +1,14 @@ #!/bin/bash -echo "deb http://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list -gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 -gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - -apt-get update +apt update -qq +# install two helper packages we need +apt install -y --no-install-recommends software-properties-common dirmngr +# add the signing key (by Michael Rutter) for these repos +# To verify key, run gpg --show-keys /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +# Fingerprint: E298A3A825C0D65DFD57CBB651716619E084DAB9 +wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +# add the R 4.X repo from CRAN +add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" apt-get update @@ -16,7 +21,6 @@ RUNDEPS="ca-certificates \ libxml2-dev \ vim-tiny \ wget \ - dirmngr \ libmagick++-dev \ libpoppler-cpp-dev \ libudunits2-dev \ @@ -28,7 +32,6 @@ RUNDEPS="ca-certificates \ libfribidi-dev \ curl \ libgit2-dev \ - pandoc-citeproc \ qpdf" # Install R amd dependencies @@ -59,7 +62,7 @@ echo 'options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRv Rscript -e "install.packages('docopt', repos= '$R_REPOS')" # Install alternative r console -pip3 install -U --no-cache-dir radian +pip3 install --user --no-cache-dir --break-system-packages radian # R packages on RSPM install2.r --error --skipinstalled --ncpus 32 \ @@ -70,7 +73,7 @@ installGithub.r \ $(grep -o '^[^#]*' tmp/r_packages_github.txt | tr '\n' ' ') # Miniconda for Refinitiv and resp. python dependenies -R -e "Refinitiv::install_eikon()" +# R -e "Refinitiv::install_eikon()" chown --recursive $USERNAME:$USERNAME /usr/local/lib/R/site-library From 9492e9bd70209318384721b59e4333866196052d Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 19:00:09 +0200 Subject: [PATCH 06/16] Fix cargo paths in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa0f614..dac4adc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,7 +141,7 @@ RUN chmod +x /tmp/install_latex.sh &&\ # Set Latex Paths ENV PATH="/usr/local/texlive/bin/x86_64-linux:${PATH}" -ENV PATH="/home/${USERNAME}/.cargo/bin:${PATH}" +ENV PATH="/root/.cargo/bin:${PATH}" # Install R ENV R_VERSION=4.4.1 From 2e18bb10f4e4ca70d9e518211b8af95a06db3014 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 20:14:25 +0200 Subject: [PATCH 07/16] Fix cargo install / path --- Dockerfile | 57 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index dac4adc..d952fc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -136,42 +136,43 @@ RUN chmod +x /tmp/install_latex.sh &&\ && tlmgr option -- srcfiles 0 \ && tlmgr install \ $(grep -o '^[^#]*' /tmp/latex_packages.txt | tr '\n' ' ') \ - && chown --recursive $USERNAME:$USERNAME /usr/local/texlive \ - && cargo install tex-fmt + && chown --recursive $USERNAME:$USERNAME /usr/local/texlive # Set Latex Paths ENV PATH="/usr/local/texlive/bin/x86_64-linux:${PATH}" -ENV PATH="/root/.cargo/bin:${PATH}" # Install R ENV R_VERSION=4.4.1 # Set RSPM snapshot see: -# https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 -ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/noble/2024-10-01 - -COPY install_scripts/install_r.sh /tmp/install_r.sh -COPY package_lists/r_packages.txt /tmp/r_packages.txt -COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt - -RUN chmod +x /tmp/install_r.sh &&\ + # https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 + ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/noble/2024-10-01 + + COPY install_scripts/install_r.sh /tmp/install_r.sh + COPY package_lists/r_packages.txt /tmp/r_packages.txt + COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt + + RUN chmod +x /tmp/install_r.sh &&\ /tmp/install_r.sh - -COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. - -COPY --chown=$USERNAME .misc/.Rprofile /home/$USERNAME/. - -RUN mkdir /home/$USERNAME/.R && chown -R $USERNAME /home/$USERNAME/.R -COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. - -RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache -COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. - -RUN chown -R $USERNAME /usr/local/lib -RUN chown -R $USERNAME /usr/local/include - -# Switch to non-root user -USER $USERNAME - + + COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. + + COPY --chown=$USERNAME .misc/.Rprofile /home/$USERNAME/. + + RUN mkdir /home/$USERNAME/.R && chown -R $USERNAME /home/$USERNAME/.R + COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. + + RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache + COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. + + RUN chown -R $USERNAME /usr/local/lib + RUN chown -R $USERNAME /usr/local/include + + # Switch to non-root user + USER $USERNAME + + RUN cargo install tex-fmt + ENV PATH="/home/${USERNAME}/.cargo/bin:${PATH}" + # Start zsh CMD [ "zsh" ] \ No newline at end of file From 60a506801a64856097c5565c04492b232902e77f Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 21:14:33 +0200 Subject: [PATCH 08/16] Adjust PATH --- Dockerfile | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index d952fc9..d96bb4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,34 +145,34 @@ ENV PATH="/usr/local/texlive/bin/x86_64-linux:${PATH}" ENV R_VERSION=4.4.1 # Set RSPM snapshot see: - # https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 - ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/noble/2024-10-01 - - COPY install_scripts/install_r.sh /tmp/install_r.sh - COPY package_lists/r_packages.txt /tmp/r_packages.txt - COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt - - RUN chmod +x /tmp/install_r.sh &&\ - /tmp/install_r.sh - - COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. - - COPY --chown=$USERNAME .misc/.Rprofile /home/$USERNAME/. - - RUN mkdir /home/$USERNAME/.R && chown -R $USERNAME /home/$USERNAME/.R - COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. - - RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache - COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. - - RUN chown -R $USERNAME /usr/local/lib - RUN chown -R $USERNAME /usr/local/include - - # Switch to non-root user - USER $USERNAME - - RUN cargo install tex-fmt - ENV PATH="/home/${USERNAME}/.cargo/bin:${PATH}" +# https://packagemanager.posit.co/client/#/repos/cran/setup?r_environment=other&snapshot=2024-10-01&distribution=ubuntu-22.04 +ENV R_REPOS=https://packagemanager.posit.co/cran/__linux__/noble/2024-10-01 + +COPY install_scripts/install_r.sh /tmp/install_r.sh +COPY package_lists/r_packages.txt /tmp/r_packages.txt +COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt + +RUN chmod +x /tmp/install_r.sh &&\ +/tmp/install_r.sh + +COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. + +COPY --chown=$USERNAME .misc/.Rprofile /home/$USERNAME/. + +RUN mkdir /home/$USERNAME/.R && chown -R $USERNAME /home/$USERNAME/.R +COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. + +RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache +COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. + +RUN chown -R $USERNAME /usr/local/lib +RUN chown -R $USERNAME /usr/local/include + +# Switch to non-root user +USER $USERNAME + +RUN cargo install tex-fmt +ENV PATH="/home/ubuntu/.cargo/bin:${PATH}" # Start zsh CMD [ "zsh" ] \ No newline at end of file From 0e1cd2a103dbb903d4deeb18e47a35a8b3160c7d Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 2 Oct 2024 22:00:48 +0200 Subject: [PATCH 09/16] Update python packages path --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d96bb4a..e2a891b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -165,6 +165,8 @@ COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. +ENV PATH="/root/.local/bin:${PATH}" +RUN chown -R $USERNAME /root/.local/bin RUN chown -R $USERNAME /usr/local/lib RUN chown -R $USERNAME /usr/local/include From 45cf375e08c80eeee1938e401b380b2355f8ef9a Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Tue, 8 Oct 2024 11:02:09 +0200 Subject: [PATCH 10/16] Use python venv --- .devcontainer/devcontainer.json | 5 +++-- Dockerfile | 32 ++++++++++++++--------------- install_scripts/install_r.sh | 7 ++----- package_lists/python_packages.txt | 1 + package_lists/r_packages.txt | 1 + package_lists/r_packages_github.txt | 1 - 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7f0ad21..08d61ee 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -65,11 +65,12 @@ }, // Python "jupyter.notebookFileRoot": "${workspaceFolder}", - "python.pythonPath": "/usr/bin/python3", + "python.pythonPath": "/opt/venv/python", + "python.defaultInterpreterPath": "/opt/venv/python", "python.dataScience.interactiveWindowMode": "perFile", "python.dataScience.sendSelectionToInteractiveWindow": true, // R - "r.rterm.linux": "/usr/local/bin/radian", + "r.rterm.linux": "/opt/venv/radian", "r.bracketedPaste": true, "r.sessionWatcher": true, "r.plot.useHttpgd": true, diff --git a/Dockerfile b/Dockerfile index e2a891b..3b79be8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ SHELL ["/bin/bash", "-c"] ENV DISPLAY=:0 \ TZ=Europe/Berlin +ARG VIRTUAL_ENV=/opt/venv ARG USERNAME=ubuntu ARG USER_UID=1000 ARG USER_GID=$USER_UID @@ -47,6 +48,9 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele ssh-client \ fontconfig \ pkg-config \ + python3-pip \ + python3-dev \ + python3-venv \ default-libmysqlclient-dev \ ttf-mscorefonts-installer \ locales &&\ @@ -56,6 +60,11 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele git clone --depth=1 https://github.com/sindresorhus/pure.git /home/$USERNAME/.zsh/pure \ && rm -rf /home/$USERNAME/.zsh/pure/.git \ && apt-get autoclean -y \ + && apt-get clean \ + && apt-get autoclean \ + && rm -rf /var/cache/* \ + && rm -rf /tmp/* \ + && rm -rf /var/tmp/* \ && rm -rf /var/lib/apt/lists/* ENV LC_ALL=en_US.UTF-8 \ @@ -108,22 +117,16 @@ RUN git clone --depth=1 https://github.com/RUrlus/carma.git /usr/local/carma \ && rm -rf /usr/local/carma # Install Python +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY package_lists/python_packages.txt /package_lists/python_packages.txt -RUN apt-get update &&\ - apt-get -y --no-install-recommends install \ - python3-pip \ - python3-dev \ - python3-venv && \ - # Python packages - pip3 install -U --no-cache-dir --break-system-packages \ +RUN pip install --upgrade pip \ + && pip3 install \ $(grep -o '^[^#]*' package_lists/python_packages.txt | tr '\n' ' ') \ && apt-get autoclean -y \ && rm -rf /var/lib/apt/lists/* -# Set PATH for user installed python packages -ENV PATH="/home/${USERNAME}/.local/bin:${PATH}" - # Install Latex COPY install_scripts/install_latex.sh /tmp/install_latex.sh COPY package_lists/latex_packages.txt /tmp/latex_packages.txt @@ -153,7 +156,7 @@ COPY package_lists/r_packages.txt /tmp/r_packages.txt COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt RUN chmod +x /tmp/install_r.sh &&\ -/tmp/install_r.sh + /tmp/install_r.sh COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. @@ -165,16 +168,13 @@ COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. -ENV PATH="/root/.local/bin:${PATH}" -RUN chown -R $USERNAME /root/.local/bin -RUN chown -R $USERNAME /usr/local/lib -RUN chown -R $USERNAME /usr/local/include +RUN chown -R $USERNAME /usr/local # Switch to non-root user USER $USERNAME RUN cargo install tex-fmt ENV PATH="/home/ubuntu/.cargo/bin:${PATH}" - + # Start zsh CMD [ "zsh" ] \ No newline at end of file diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index b701c77..e2c62de 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -61,9 +61,6 @@ echo 'options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRv # Install docopt which is used by littler to install packages Rscript -e "install.packages('docopt', repos= '$R_REPOS')" -# Install alternative r console -pip3 install --user --no-cache-dir --break-system-packages radian - # R packages on RSPM install2.r --error --skipinstalled --ncpus 32 \ $(grep -o '^[^#]*' tmp/r_packages.txt | tr '\n' ' ') @@ -77,6 +74,6 @@ installGithub.r \ chown --recursive $USERNAME:$USERNAME /usr/local/lib/R/site-library -apt-get autoclean -y -rm -rf /var/lib/apt/lists/* +# apt-get autoclean -y +# rm -rf /var/lib/apt/lists/* diff --git a/package_lists/python_packages.txt b/package_lists/python_packages.txt index 235bffe..5bdc970 100644 --- a/package_lists/python_packages.txt +++ b/package_lists/python_packages.txt @@ -31,6 +31,7 @@ Pyarrow pylint pyparsing==2.4.7 pytest +radian redis requests ruff diff --git a/package_lists/r_packages.txt b/package_lists/r_packages.txt index f517b6e..0999e18 100644 --- a/package_lists/r_packages.txt +++ b/package_lists/r_packages.txt @@ -51,6 +51,7 @@ gt Hmisc htmltools htmlwidgets +httpgd igraph inline inspectdf diff --git a/package_lists/r_packages_github.txt b/package_lists/r_packages_github.txt index 33428a7..a93dfa8 100644 --- a/package_lists/r_packages_github.txt +++ b/package_lists/r_packages_github.txt @@ -2,6 +2,5 @@ BerriJ/strsplit.fix GreenGrassBlueOcean/RefinitivR jhelvy/renderthis ManuelHentschel/vscDebugger@*release -nx10/httpgd sjp/grConvert stenevang/sftp From e17da0e12b8dfb84cbd9aab765d7143f2ba79c88 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Tue, 8 Oct 2024 11:58:59 +0200 Subject: [PATCH 11/16] Add cleanup of R --- Dockerfile | 1 - install_scripts/install_r.sh | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b79be8..4679d0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,6 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele && rm -rf /home/$USERNAME/.zsh/pure/.git \ && apt-get autoclean -y \ && apt-get clean \ - && apt-get autoclean \ && rm -rf /var/cache/* \ && rm -rf /tmp/* \ && rm -rf /var/tmp/* \ diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index e2c62de..5657a8f 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -74,6 +74,9 @@ installGithub.r \ chown --recursive $USERNAME:$USERNAME /usr/local/lib/R/site-library -# apt-get autoclean -y -# rm -rf /var/lib/apt/lists/* - +apt-get autoclean -y +apt-get clean +rm -rf /var/cache/* +rm -rf /tmp/* +rm -rf /var/tmp/* +rm -rf /var/lib/apt/lists/* \ No newline at end of file From 96ca106853ade75523fd3ad37e84e7b1029ca364 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Tue, 8 Oct 2024 14:31:39 +0200 Subject: [PATCH 12/16] Minor refinements in Dockerfile --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4679d0c..f573295 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,6 @@ COPY .misc/lmroman10-regular-webfont.ttf /usr/share/fonts/truetype/. COPY .misc/lmroman10-italic-webfont.ttf /usr/share/fonts/truetype/. COPY .misc/lmroman10-bolditalic-webfont.ttf /usr/share/fonts/truetype/. COPY .misc/lmroman10-bold-webfont.ttf /usr/share/fonts/truetype/. - RUN fc-cache -f -v # Install quarto @@ -157,6 +156,8 @@ COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt RUN chmod +x /tmp/install_r.sh &&\ /tmp/install_r.sh +RUN chown -R $USERNAME /usr/local + COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. COPY --chown=$USERNAME .misc/.Rprofile /home/$USERNAME/. @@ -167,8 +168,6 @@ COPY --chown=$USERNAME .misc/Makevars /home/$USERNAME/.R/. RUN mkdir /home/$USERNAME/.ccache && chown -R $USERNAME /home/$USERNAME/.ccache COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. -RUN chown -R $USERNAME /usr/local - # Switch to non-root user USER $USERNAME From ddf5d01c152137095e15bb8c9fad2aa338f5b3eb Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 9 Oct 2024 10:41:49 +0200 Subject: [PATCH 13/16] Install python packages as $USER --- Dockerfile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index f573295..49b70de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,17 +114,6 @@ RUN git clone --depth=1 https://github.com/RUrlus/carma.git /usr/local/carma \ && cmake --build . --config Release --target install \ && rm -rf /usr/local/carma -# Install Python -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -COPY package_lists/python_packages.txt /package_lists/python_packages.txt - -RUN pip install --upgrade pip \ - && pip3 install \ - $(grep -o '^[^#]*' package_lists/python_packages.txt | tr '\n' ' ') \ - && apt-get autoclean -y \ - && rm -rf /var/lib/apt/lists/* - # Install Latex COPY install_scripts/install_latex.sh /tmp/install_latex.sh COPY package_lists/latex_packages.txt /tmp/latex_packages.txt @@ -171,6 +160,15 @@ COPY --chown=$USERNAME .misc/ccache.conf /home/$USERNAME/.ccache/. # Switch to non-root user USER $USERNAME +# Install Python Packages +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +COPY package_lists/python_packages.txt /package_lists/python_packages.txt + +RUN pip install --upgrade pip \ + && pip3 install \ + $(grep -o '^[^#]*' package_lists/python_packages.txt | tr '\n' ' ') + RUN cargo install tex-fmt ENV PATH="/home/ubuntu/.cargo/bin:${PATH}" From 4e9da65bd475d7a2c23fb33b882761e4ac546ad7 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 9 Oct 2024 11:15:08 +0200 Subject: [PATCH 14/16] Update python venv location --- Dockerfile | 2 +- package_lists/r_packages.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49b70de..c9baa6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ SHELL ["/bin/bash", "-c"] ENV DISPLAY=:0 \ TZ=Europe/Berlin -ARG VIRTUAL_ENV=/opt/venv ARG USERNAME=ubuntu +ARG VIRTUAL_ENV=/home/$USERNAME/python/venv ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG DEBIAN_FRONTEND=noninteractive diff --git a/package_lists/r_packages.txt b/package_lists/r_packages.txt index 0999e18..24aac25 100644 --- a/package_lists/r_packages.txt +++ b/package_lists/r_packages.txt @@ -30,6 +30,7 @@ fontawesome foreach forecast gamlss.dist +MixGHD gamlss.lasso GAS gdata From 71feb93e9abbdabac51f2f87ee16595a422f951b Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 9 Oct 2024 12:08:53 +0200 Subject: [PATCH 15/16] Install R packages to /home/$USERNAME/R/library --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c9baa6a..e2f7bc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ENV DISPLAY=:0 \ TZ=Europe/Berlin ARG USERNAME=ubuntu +ENV R_LIBS_USER=/home/$USERNAME/R/library ARG VIRTUAL_ENV=/home/$USERNAME/python/venv ARG USER_UID=1000 ARG USER_GID=$USER_UID @@ -18,6 +19,7 @@ RUN usermod -a -G staff,$USERNAME $USERNAME # Create folders to mount extensions RUN mkdir -p /home/$USERNAME/.vscode-server/extensions \ /home/$USERNAME/.vscode-server-insiders/extensions \ + /home/$USERNAME/R/library \ workspaces \ && chown -R $USERNAME \ /home/$USERNAME/.vscode-server \ @@ -145,7 +147,8 @@ COPY package_lists/r_packages_github.txt /tmp/r_packages_github.txt RUN chmod +x /tmp/install_r.sh &&\ /tmp/install_r.sh -RUN chown -R $USERNAME /usr/local +RUN chown -R $USERNAME /usr/local/lib +RUN chown -R $USERNAME /usr/local/include COPY --chown=$USERNAME .misc/.zshrc /home/$USERNAME/. @@ -166,7 +169,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY package_lists/python_packages.txt /package_lists/python_packages.txt RUN pip install --upgrade pip \ - && pip3 install \ + && pip --no-cache-dir install \ $(grep -o '^[^#]*' package_lists/python_packages.txt | tr '\n' ' ') RUN cargo install tex-fmt From a77185d287a20ebf1329da1b4000f28975c5b945 Mon Sep 17 00:00:00 2001 From: Jonathan Berrisch Date: Wed, 9 Oct 2024 12:49:36 +0200 Subject: [PATCH 16/16] Fix R permissions --- install_scripts/install_r.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts/install_r.sh b/install_scripts/install_r.sh index 5657a8f..c819d02 100644 --- a/install_scripts/install_r.sh +++ b/install_scripts/install_r.sh @@ -72,7 +72,7 @@ installGithub.r \ # Miniconda for Refinitiv and resp. python dependenies # R -e "Refinitiv::install_eikon()" -chown --recursive $USERNAME:$USERNAME /usr/local/lib/R/site-library +chown --recursive $USERNAME:$USERNAME /home/$USERNAME/R/library apt-get autoclean -y apt-get clean