chore: Added build container, devcontainer, and vscode project settings

.devcontainer/devcontainer.json

@@ -0,0 +1,19 @@
+{
+    "image": "synapseml-host",
+    "remoteUser": "root",
+    "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.black-formatter",
+        "GitHub.copilot",
+        "scala-lang.scala",
+        "itryapitsin.Sbt",
+        "scalameta.metals"
+      ]
+    }
+  },
+  "mounts": [
+      "source=${localEnv:HOME}${localEnv:USERPROFILE}/.azure,target=/home/dev/.azure/,type=bind,consistency=cached"
+    ]
+}

.gitignore

@@ -21,7 +21,6 @@
 *.iml
 
 # Generic editors
-.vscode
 .idea
 .metals
 .bloop
@@ -88,3 +87,5 @@ metastore_db/
 **/dist/*
 **/*.egg-info/*
 
+# build container
+!tools/docker/build/*

.vscode/extensions.json

@@ -0,0 +1,12 @@
+{
+	"recommendations": [
+        "ms-python.python",
+        "ms-python.black-formatter",
+        "GitHub.copilot",
+        "scala-lang.scala",
+        "itryapitsin.Sbt",
+        "scalameta.metals"
+	],
+	"unwantedRecommendations": [
+	]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.watcherExclude": {
+        "**/target": true
+    },
+    "azure-pipelines.1ESPipelineTemplatesSchemaFile": true
+}

environment.yml

@@ -1,7 +1,6 @@
 name: synapseml
 channels:
   - conda-forge
-  - default
 dependencies:
   - python=3.11.8
   - requests=2.26.0
@@ -10,44 +9,48 @@ dependencies:
   - r-sparklyr=1.8.1
   - r-devtools=2.4.2
   - pip:
-    - pyarrow>=0.15.0
+    - azure-cli==2.60.0
+    - azure-storage-blob==12.20.0
+    - black[jupyter]==22.3.0
+    - black==22.3.0
+    - chardet==5.2.0
+    - conda-pack==0.7.1
+    - coverage==7.5.1
+    - flake8==7.0.0
+    - horovod==0.28.1
+    - huggingface-hub==0.23.0
+    - ipython==8.24.0
+    - jupyter==1.0.0
+    - langchain==0.0.152
+    - markdownify==0.12.1
+    - matplotlib==3.9.0
+    - mistletoe==1.3.0
+    - mlflow==2.12.2
+    - nbconvert==7.16.4
+    - nbformat==5.10.4
+    - numpy==1.26.4
+    - onnxmltools==1.7.0
+    - openai==0.27.5
+    - opencv-python==4.9.0.80
+    - pandas==1.5.0
+    - petastorm==0.12.1
+    - Pillow==10.3.0
+    - pyarrow==15.0.2
+    - PyGithub==1.59.1
+    - pypandoc==1.13
     - pyspark==3.4.1
-    - pandas==1.4.0
-    - wheel
-    - sphinx==5.0.2
+    - pytest-codeblocks==0.17.0
+    - pytest-cov==5.0.0
+    - pytest==8.2.0
+    - pytorch_lightning==1.5.0
+    - PyYAML==6.0.1
     - sphinx_paramlinks==0.5.2
-    - sphinx_rtd_theme
-    - coverage
-    - pytest
-    - pytest-cov
-    - nbconvert
-    - nbformat
-    - pyyaml
-    - PyGithub
-    - tqdm
-    - ipython
-    - pytest-codeblocks
-    - azure-storage-blob
-    - twine
-    - jupyter
-    - mlflow
-    - numpy
+    - sphinx_rtd_theme==2.0.0
+    - sphinx==5.0.2
     - torch==2.0.0
     - torchvision==0.15.1
-    - horovod==0.28.1
-    - petastorm>=0.11.0
-    - pytorch_lightning==1.5.0
-    - onnxmltools==1.7.0
-    - matplotlib
-    - Pillow
+    - tqdm==4.66.4
+    - traitlets==5.14.3
     - transformers==4.32.1
-    - huggingface-hub>=0.8.1
-    - langchain==0.0.152
-    - openai==0.27.5
-    - black==22.3.0
-    - black[jupyter]==22.3.0
-    - mistletoe
-    - pypandoc
-    - markdownify
-    - traitlets
-    - opencv-python
+    - twine==5.0.0
+    - wheel==0.43.0

tools/docker/build/Dockerfile

@@ -0,0 +1,34 @@
+FROM mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+
+USER root
+
+SHELL ["/bin/bash", "--login", "-c"]
+
+RUN mkdir -p /etc/micromamba
+
+RUN wget -qO- https://aka.ms/install-artifacts-credprovider.sh | bash && \
+    curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \
+    ./bin/micromamba shell init -s bash -p ~/micromamba  # this writes to your .bashrc file && \
+    ./bin/micromamba config append channels conda-forge && \
+    ./bin/micromamba config set channel_priority strict
+
+RUN curl -fL https://github.com/coursier/coursier/releases/latest/download/cs-x86_64-pc-linux.gz | gzip -d > cs && chmod +x cs && ./cs setup --apps scala,scalac,sbt --yes && \
+    echo "export PATH=$PATH:/root/.local/share/coursier/bin" >> ~/.bashrc
+
+RUN tdnf -y update && \
+    tdnf -y install msopenjdk-11 vi conda && \
+    python3.9 -m pip install -U --force-reinstall charset-normalizer  && \
+    python3.9 -m pip install conda-pack && \
+    tdnf clean all 
+
+RUN git config --global core.autocrlf true
+
+COPY environment.yml /etc/micromamba/environment.yml
+RUN micromamba env create -f /etc/micromamba/environment.yml -y
+
+COPY website/environment.yml /etc/micromamba/website_environment.yml
+RUN micromamba env create -f /etc/micromamba/website_environment.yml -y
+
+RUN echo "micromamba activate synapseml" >> ~/.bashrc
+
+WORKDIR /

tools/docker/build/build.ps1

@@ -0,0 +1,5 @@
+#!/usr/bin/env pwsh
+$ScriptPath = $MyInvocation.MyCommand.Path
+$ScriptDir = Split-Path -Parent $ScriptPath
+$DockerContext = Resolve-Path "$ScriptDir\..\..\.."
+docker build -t synapseml-host -f "$ScriptDir\Dockerfile" $DockerContext

tools/docker/build/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
+DOCKER_CONTEXT=$(realpath "$SCRIPT_DIR/../../..")
+docker build -t synapseml-host -f "$SCRIPT_DIR/Dockerfile" "$DOCKER_CONTEXT"

website/README.md

@@ -2,6 +2,14 @@
 
 This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator.
 
+### Environment
+
+```
+$ conda env create -f website/environment.yml
+$ conda activate synapseml-website
+$ cd website
+```
+
 ### Installation
 
 ```

website/environment.yml

@@ -0,0 +1,5 @@
+name: synapseml-website
+channels:
+  - conda-forge
+dependencies:
+  - conda-forge::yarn=1.22.19