From 046c9dc89a1fb4d847819949416624c16aa5b90b Mon Sep 17 00:00:00 2001 From: Brendan Walsh Date: Thu, 16 May 2024 11:50:40 +0000 Subject: [PATCH 1/5] chore: Added build container, devcontainer, and vscode project settings --- .devcontainer/devcontainer.json | 19 ++++++++ .gitignore | 1 - .vscode/extensions.json | 12 +++++ .vscode/settings.json | 6 +++ environment.yml | 77 +++++++++++++++++---------------- website/README.md | 8 ++++ website/environment.yml | 5 +++ 7 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 website/environment.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..95b846189b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,19 @@ +{ + "image": "synapseml-host", + "remoteUser": "root", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.black-formatter", + "GitHub.copilot", + "scala-lang.scala", + "itryapitsin.Sbt", + "scalameta.metals" + ] + } + }, + "mounts": [ + "source=${localEnv:HOME}${localEnv:USERPROFILE}/.azure,target=/home/dev/.azure/,type=bind,consistency=cached" + ] +} diff --git a/.gitignore b/.gitignore index d2a5a21ab6..2ee3576f48 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ *.iml # Generic editors -.vscode .idea .metals .bloop diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000..234118947a --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,12 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.black-formatter", + "GitHub.copilot", + "scala-lang.scala", + "itryapitsin.Sbt", + "scalameta.metals" + ], + "unwantedRecommendations": [ + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..de00ac6cb0 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "files.watcherExclude": { + "**/target": true + }, + "azure-pipelines.1ESPipelineTemplatesSchemaFile": true +} \ No newline at end of file diff --git a/environment.yml b/environment.yml index 5302760212..7d36ad102a 100644 --- a/environment.yml +++ b/environment.yml @@ -1,7 +1,6 @@ name: synapseml channels: - conda-forge - - default dependencies: - python=3.11.8 - requests=2.26.0 @@ -10,44 +9,48 @@ dependencies: - r-sparklyr=1.8.1 - r-devtools=2.4.2 - pip: - - pyarrow>=0.15.0 + - azure-cli==2.60.0 + - azure-storage-blob==12.20.0 + - black[jupyter]==22.3.0 + - black==22.3.0 + - chardet==5.2.0 + - conda-pack==0.7.1 + - coverage==7.5.1 + - flake8==7.0.0 + - horovod==0.28.1 + - huggingface-hub==0.23.0 + - ipython==8.24.0 + - jupyter==1.0.0 + - langchain==0.0.152 + - markdownify==0.12.1 + - matplotlib==3.9.0 + - mistletoe==1.3.0 + - mlflow==2.12.2 + - nbconvert==7.16.4 + - nbformat==5.10.4 + - numpy==1.26.4 + - onnxmltools==1.7.0 + - openai==0.27.5 + - opencv-python==4.9.0.80 + - pandas==1.5.0 + - petastorm==0.12.1 + - Pillow==10.3.0 + - pyarrow==15.0.2 + - PyGithub==1.59.1 + - pypandoc==1.13 - pyspark==3.4.1 - - pandas==1.4.0 - - wheel - - sphinx==5.0.2 + - pytest-codeblocks==0.17.0 + - pytest-cov==5.0.0 + - pytest==8.2.0 + - pytorch_lightning==1.5.0 + - PyYAML==6.0.1 - sphinx_paramlinks==0.5.2 - - sphinx_rtd_theme - - coverage - - pytest - - pytest-cov - - nbconvert - - nbformat - - pyyaml - - PyGithub - - tqdm - - ipython - - pytest-codeblocks - - azure-storage-blob - - twine - - jupyter - - mlflow - - numpy + - sphinx_rtd_theme==2.0.0 + - sphinx==5.0.2 - torch==2.0.0 - torchvision==0.15.1 - - horovod==0.28.1 - - petastorm>=0.11.0 - - pytorch_lightning==1.5.0 - - onnxmltools==1.7.0 - - matplotlib - - Pillow + - tqdm==4.66.4 + - traitlets==5.14.3 - transformers==4.32.1 - - huggingface-hub>=0.8.1 - - langchain==0.0.152 - - openai==0.27.5 - - black==22.3.0 - - black[jupyter]==22.3.0 - - mistletoe - - pypandoc - - markdownify - - traitlets - - opencv-python + - twine==5.0.0 + - wheel==0.43.0 \ No newline at end of file diff --git a/website/README.md b/website/README.md index e1299a266f..e596bbb874 100644 --- a/website/README.md +++ b/website/README.md @@ -2,6 +2,14 @@ This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator. +### Environment + +``` +$ conda env create -f website/environment.yml +$ conda activate synapseml-website +$ cd website +``` + ### Installation ``` diff --git a/website/environment.yml b/website/environment.yml new file mode 100644 index 0000000000..6beaf5f278 --- /dev/null +++ b/website/environment.yml @@ -0,0 +1,5 @@ +name: synapseml-website +channels: + - conda-forge +dependencies: + - conda-forge::yarn=1.22.19 From d87c6b27667551e71701c3703f1db6bc66680a35 Mon Sep 17 00:00:00 2001 From: Brendan Walsh Date: Thu, 16 May 2024 11:52:50 +0000 Subject: [PATCH 2/5] whitespace --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7d36ad102a..a0813ca78b 100644 --- a/environment.yml +++ b/environment.yml @@ -53,4 +53,4 @@ dependencies: - traitlets==5.14.3 - transformers==4.32.1 - twine==5.0.0 - - wheel==0.43.0 \ No newline at end of file + - wheel==0.43.0 From 6ca49ad9e4d0e1b721ebd2c972db0aa2c12e9b9f Mon Sep 17 00:00:00 2001 From: Brendan Walsh Date: Thu, 16 May 2024 11:56:05 +0000 Subject: [PATCH 3/5] added dockerfile --- .gitignore | 2 ++ tools/docker/build/Dockerfile | 40 +++++++++++++++++++++++++++++++++++ tools/docker/build/build.ps1 | 5 +++++ tools/docker/build/build.sh | 5 +++++ 4 files changed, 52 insertions(+) create mode 100644 tools/docker/build/Dockerfile create mode 100644 tools/docker/build/build.ps1 create mode 100644 tools/docker/build/build.sh diff --git a/.gitignore b/.gitignore index 2ee3576f48..5edef251de 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ metastore_db/ **/dist/* **/*.egg-info/* +# build container +!tools/docker/build/* diff --git a/tools/docker/build/Dockerfile b/tools/docker/build/Dockerfile new file mode 100644 index 0000000000..98eea58256 --- /dev/null +++ b/tools/docker/build/Dockerfile @@ -0,0 +1,40 @@ +FROM mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + +USER root + +# Make sure bash commands are run with .bashrc +SHELL ["/bin/bash", "--login", "-c"] + +RUN mkdir -p /etc/micromamba + +# Install NuGet credential provider (ttps://github.com/microsoft/artifacts-credprovider#setup) +RUN wget -qO- https://aka.ms/install-artifacts-credprovider.sh | bash && \ + curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \ + ./bin/micromamba shell init -s bash -p ~/micromamba # this writes to your .bashrc file && \ + ./bin/micromamba config append channels conda-forge && \ + ./bin/micromamba config set channel_priority strict + +RUN curl -fL https://github.com/coursier/coursier/releases/latest/download/cs-x86_64-pc-linux.gz | gzip -d > cs && chmod +x cs && ./cs setup --apps scala,scalac,sbt --yes && \ + echo "export PATH=$PATH:/root/.local/share/coursier/bin" >> ~/.bashrc + +# procps-ng has "ps" +RUN tdnf -y update && \ + tdnf -y install msopenjdk-11 vi conda && \ + python3.9 -m pip install -U --force-reinstall charset-normalizer && \ + python3.9 -m pip install conda-pack && \ + tdnf clean all + +RUN git config --global core.autocrlf true + +# RUN pip3 uninstall -y urllib3 && \ +# pip3 install requests + +COPY environment.yml /etc/micromamba/environment.yml +RUN micromamba env create -f /etc/micromamba/environment.yml -y + +COPY website/environment.yml /etc/micromamba/website_environment.yml +RUN micromamba env create -f /etc/micromamba/website_environment.yml -y + +RUN echo "micromamba activate synapseml" >> ~/.bashrc + +WORKDIR / \ No newline at end of file diff --git a/tools/docker/build/build.ps1 b/tools/docker/build/build.ps1 new file mode 100644 index 0000000000..9a6d9da80b --- /dev/null +++ b/tools/docker/build/build.ps1 @@ -0,0 +1,5 @@ +#!/usr/bin/env pwsh +$ScriptPath = $MyInvocation.MyCommand.Path +$ScriptDir = Split-Path -Parent $ScriptPath +$DockerContext = Resolve-Path "$ScriptDir\..\..\.." +docker build -t synapseml-host -f "$ScriptDir\Dockerfile" $DockerContext diff --git a/tools/docker/build/build.sh b/tools/docker/build/build.sh new file mode 100644 index 0000000000..2a23c1f65b --- /dev/null +++ b/tools/docker/build/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +SCRIPT_PATH=$(realpath "$0") +SCRIPT_DIR=$(dirname "$SCRIPT_PATH") +DOCKER_CONTEXT=$(realpath "$SCRIPT_DIR/../../..") +docker build -t my_image_name -f "$SCRIPT_DIR/Dockerfile" "$DOCKER_CONTEXT" From bc46f50e3b2fe52ef7b2ae78c1aedabb9134a247 Mon Sep 17 00:00:00 2001 From: Brendan Walsh Date: Thu, 16 May 2024 12:02:15 +0000 Subject: [PATCH 4/5] cleaned up Dockerfile --- tools/docker/build/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/docker/build/Dockerfile b/tools/docker/build/Dockerfile index 98eea58256..bacf275d78 100644 --- a/tools/docker/build/Dockerfile +++ b/tools/docker/build/Dockerfile @@ -2,12 +2,10 @@ FROM mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 USER root -# Make sure bash commands are run with .bashrc SHELL ["/bin/bash", "--login", "-c"] RUN mkdir -p /etc/micromamba -# Install NuGet credential provider (ttps://github.com/microsoft/artifacts-credprovider#setup) RUN wget -qO- https://aka.ms/install-artifacts-credprovider.sh | bash && \ curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \ ./bin/micromamba shell init -s bash -p ~/micromamba # this writes to your .bashrc file && \ @@ -17,7 +15,6 @@ RUN wget -qO- https://aka.ms/install-artifacts-credprovider.sh | bash && \ RUN curl -fL https://github.com/coursier/coursier/releases/latest/download/cs-x86_64-pc-linux.gz | gzip -d > cs && chmod +x cs && ./cs setup --apps scala,scalac,sbt --yes && \ echo "export PATH=$PATH:/root/.local/share/coursier/bin" >> ~/.bashrc -# procps-ng has "ps" RUN tdnf -y update && \ tdnf -y install msopenjdk-11 vi conda && \ python3.9 -m pip install -U --force-reinstall charset-normalizer && \ @@ -26,9 +23,6 @@ RUN tdnf -y update && \ RUN git config --global core.autocrlf true -# RUN pip3 uninstall -y urllib3 && \ -# pip3 install requests - COPY environment.yml /etc/micromamba/environment.yml RUN micromamba env create -f /etc/micromamba/environment.yml -y From 7ff6fcb66674da4f3dcf619602f576c9082a7cec Mon Sep 17 00:00:00 2001 From: Brendan Walsh Date: Thu, 16 May 2024 12:03:51 +0000 Subject: [PATCH 5/5] fix image name in linux build script --- tools/docker/build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build/build.sh b/tools/docker/build/build.sh index 2a23c1f65b..ccbffa964e 100644 --- a/tools/docker/build/build.sh +++ b/tools/docker/build/build.sh @@ -2,4 +2,4 @@ SCRIPT_PATH=$(realpath "$0") SCRIPT_DIR=$(dirname "$SCRIPT_PATH") DOCKER_CONTEXT=$(realpath "$SCRIPT_DIR/../../..") -docker build -t my_image_name -f "$SCRIPT_DIR/Dockerfile" "$DOCKER_CONTEXT" +docker build -t synapseml-host -f "$SCRIPT_DIR/Dockerfile" "$DOCKER_CONTEXT"