diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index af80dfb72..4a5d9f57a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,12 +1,19 @@ -FROM ubuntu:24.04 AS spython-base +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /workspace + RUN export DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y --no-install-recommends git gh RUN apt-get install -y --no-install-recommends less RUN apt-get install -y --no-install-recommends python3 -RUN apt-get install -y --no-install-recommends python3.12-venv python3-pip +RUN apt-get install -y --no-install-recommends python3.12-venv +RUN apt-get install -y --no-install-recommends python3-pip RUN apt-get install -y --no-install-recommends build-essential -RUN apt-get install -y --no-install-recommends ruby ruby-dev +RUN apt-get install -y --no-install-recommends ruby +RUN apt-get install -y --no-install-recommends ruby-dev RUN apt-get install -y --no-install-recommends bundler RUN apt-get install -y --no-install-recommends nodejs RUN apt-get install -y --no-install-recommends npm @@ -14,4 +21,6 @@ RUN apt-get install -y --no-install-recommends ditaa RUN apt-get install -y --no-install-recommends libyaml-dev RUN apt-get clean autoclean RUN apt-get autoremove -y -RUN rm -rf /var/lib/{apt, dpkg, cache, log} +RUN rm -rf /var/lib/{apt,dpkg,cache,log}/* + +WORKDIR /workspace diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 73c2dfd92..c79c6e138 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,6 +26,8 @@ jobs: needs: check_date if: ${{ needs.check_date.outputs.should_run != 'false' }} runs-on: ubuntu-latest + env: + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml index da6f5b15e..5d7b86a93 100644 --- a/.github/workflows/regress.yml +++ b/.github/workflows/regress.yml @@ -13,6 +13,8 @@ jobs: - uses: pre-commit/action@v3.0.1 regress-smoke: runs-on: ubuntu-latest + env: + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 @@ -45,6 +47,7 @@ jobs: env: MANUAL_NAME: isa VERSIONS: all + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 @@ -75,6 +78,7 @@ jobs: env: EXT: B VERSION: latest + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 @@ -102,6 +106,8 @@ jobs: regress-gen-certificate: runs-on: ubuntu-latest needs: regress-smoke + env: + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 @@ -129,6 +135,8 @@ jobs: regress-gen-profile: runs-on: ubuntu-latest needs: regress-smoke + env: + SINGULARITY: 1 steps: - name: Clone Github Repo Action uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index e82076dd8..ddd1aa6a7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .stamps .venv .asciidoctor +.container-type diag-ditaa-* arch/manual/isa/**/riscv-isa-manual gen diff --git a/bin/setup b/bin/setup index 9ea0591f5..934c8033a 100755 --- a/bin/setup +++ b/bin/setup @@ -6,66 +6,146 @@ cd $ROOT CONTAINER_TAG=`cat ${ROOT}/bin/.container-tag` +# Supported container types: +# +# = devcontainer +# +# When devcontainer is used, all commands run "natively" (no container prefix), because it's assumed +# that the command itself is already in the container context +# +# = docker +# +# A docker container. All commands on the host (using ./do or ./bin/*) are run in the context of a Docker image. +# +# = singularity +# +# An apptainer/singularity container. All commands on the host (using ./do or ./bin/*) are run in the context of an Apptainer/Singularity image. +# +# +# ALL THREE CONTAINERS ARE IDENTICAL. Which one you use is a preference of your setup and installed +# software. + + +# get the container type +# +# use, in priority order: +# 1. devcontainer, if DEVCONTAINER_ENV is set in the enviornment +# 2. the type stored in .container-type +# 3. Docker, if DOCKER is set in the enviornment +# 4. Singularity, if SINGULARITY is set in the enviornment +# 5. The choice made by the user, which will be cached in .container-type +if [ -v DEVCONTAINER_ENV ]; then +CONTAINER_TYPE=devcontainer +elif [ -f ${ROOT}/.container-type ]; then +CONTAINER_TYPE=`cat ${ROOT}/.container-type` +elif [ -v DOCKER ]; then +CONTAINER_TYPE=docker +elif [ -v SINGULARITY ]; then +CONTAINER_TYPE=singularity +else + echo -e "UDB tools run in a container. Both Docker and Singularity/Apptainer are supported.\\n\\n1. Docker\\n2. Singularity\\n" + while true; do + echo "Which would you like to use? (1/2) " + read ans + case $ans in + [1]* ) CONTAINER_TYPE=docker; break;; + [2]* ) CONTAINER_TYPE=singularity; break;; + * ) echo -e "\\nPlease answer 1 or 2.";; + esac + done +fi + +if [ "${CONTAINER_TYPE}" != "docker" -a "${CONTAINER_TYPE}" != "singularity" ]; then + echo "BAD CONTAINER TYPE: ${CONTAINER_TYPE}" +fi + +if [ ! -f ${ROOT}/.container-type ]; then + echo ${CONTAINER_TYPE} > ${ROOT}/.container-type +fi + +print_env() { + if [ "${CONTAINER_TYPE}" == "docker" ]; then + echo "Using Docker environment" + else + echo "Using Singularity environment" + fi +} + if [ -v GITHUB_ACTIONS ]; then - echo "ACTIONS" - CONTAINER_PATH=${ROOT}/.singularity/image.sif - HOME_PATH=${GITHUB_WORKSPACE} - HOME_OPT="--home ${ROOT}/.home" - SINGULARITY_CACHE=--disable-cache + echo "ACTIONS" + CONTAINER_PATH=${ROOT}/.singularity/image.sif + HOME_PATH=${GITHUB_WORKSPACE} + HOME_OPT="--home ${ROOT}/.home" + SINGULARITY_CACHE=--disable-cache +elif [ "${CONTAINER_TYPE}" == "docker" ]; then + print_env + if ! docker images riscvintl/udb:${CONTAINER_TAG} | grep -q udb ; then + # TODO: pull the image if it can be found + echo "Building Docker image..." + docker build -t riscvintl/udb:${CONTAINER_TAG} -f .devcontainer/Dockerfile . + fi + if [ -t 1 -a -t 0 ]; then + DOCKER_BASE="docker run -it -v $(ROOT):$(ROOT) -w $(ROOT) riscvintl/udb:${CONTAINER_TAG}" + else + DOCKER_BASE="docker run -v $(ROOT):$(ROOT) -w $(ROOT) riscvintl/udb:${CONTAINER_TAG}" + fi + RUN="${DOCKER_BASE}" else - CONTAINER_PATH=${ROOT}/.singularity/image-$CONTAINER_TAG.sif - HOME_PATH=${HOME} - HOME_OPT="--bind ${ROOT}/.home:${HOME_PATH} --bind /local/mnt/workspace/.vscode-server:/local/mnt/workspace/.vscode-server" - SINGULARITY_CACHE= + print_env + CONTAINER_PATH=${ROOT}/.singularity/image-$CONTAINER_TAG.sif + HOME_PATH=${HOME} + HOME_OPT="--bind ${ROOT}/.home:${HOME_PATH}" + SINGULARITY_CACHE= + if [ ! -f ${CONTAINER_PATH} ]; then + echo "Fetching container..." + if [ ! -d "${ROOT}/.singularity" ]; then + mkdir -p ${ROOT}/.singularity + fi + singularity pull ${SINGULARITY_CACHE} ${CONTAINER_PATH} oras://docker.io/riscvintl/spec-generator:$CONTAINER_TAG + fi fi if [ -f $ROOT/.git ]; then - # if this is a worktree, need to add the parent git repo in, too - GIT_PATH=`git rev-parse --git-common-dir | tr -d '\n' | xargs dirname` - HOME_OPT="${HOME_OPT} --bind ${GIT_PATH}:${GIT_PATH}" + # if this is a worktree, need to add the parent git repo in, too + GIT_PATH=`git rev-parse --git-common-dir | tr -d '\n' | xargs dirname` + HOME_OPT="${HOME_OPT} --bind ${GIT_PATH}:${GIT_PATH}" fi if [ -v DEVCONTAINER_ENV ]; then - RUN="" + RUN="" +elif [ "${CONTAINER_TYPE}" == "docker" ]; then + RUN="${DOCKER_BASE}" else - RUN="singularity run ${HOME_OPT} ${CONTAINER_PATH} " + RUN="singularity run ${HOME_OPT} ${CONTAINER_PATH}" fi if [ ! -d $ROOT/.home ]; then - mkdir $ROOT/.home -fi - -if [ ! -v DEVCONTAINER_ENV ]; then - if [ ! -f ${CONTAINER_PATH} ]; then - echo "Fetching container..." - if [ ! -d "${ROOT}/.singularity" ]; then - mkdir -p ${ROOT}/.singularity - fi - singularity pull ${SINGULARITY_CACHE} ${CONTAINER_PATH} oras://docker.io/riscvintl/spec-generator:$CONTAINER_TAG - fi + mkdir $ROOT/.home fi if [ ! -f $ROOT/.bundle/config ]; then - OLDDIR=$PWD - cd $ROOT - ${RUN} bundle config set --local path ${ROOT}/.home/.gems - ${RUN} bundle config set --local cache_path ${ROOT}/.home/.cache - cd $OLDDIR + OLDDIR=$PWD + cd $ROOT + ${RUN} bundle config set --local path ${ROOT}/.home/.gems + ${RUN} bundle config set --local cache_path ${ROOT}/.home/.cache + cd $OLDDIR fi if [ ! -d $ROOT/.home/.gems ]; then - OLDDIR=$PWD - cd $ROOT - ${RUN} bundle install - cd $OLDDIR + OLDDIR=$PWD + cd $ROOT + ${RUN} bundle install + cd $OLDDIR fi if [ ! -d $ROOT/.home/.venv ]; then - ${RUN} /usr/bin/python3 -m venv ${ROOT}/.home/.venv + ${RUN} /usr/bin/python3 -m venv ${ROOT}/.home/.venv fi + source ${ROOT}/.home/.venv/bin/activate + if [ ! -f ${ROOT}/.home/.venv/bin/pre-commit ]; then - ${RUN} ${ROOT}/.home/.venv/bin/pip install -r requirements.txt + ${RUN} ${ROOT}/.home/.venv/bin/pip install -r requirements.txt fi # if [ ! -f $ROOT/ext/riscv-opcodes/README.md ]; then @@ -73,37 +153,47 @@ fi # fi if [[ ! -z "$DEVELOPMENT" && $DEVELOPMENT -eq 1 ]]; then - if [ ! -d "${ROOT}/.home/.yard/gem_index"]; then - ${RUN} bundle exec --gemfile ${ROOT}/Gemfile yard config --gem-install-yri - ${RUN} bundle exec --gemfile ${ROOT}/Gemfile yard gems - touch ${ROOT}/.stamps/dev_gems - fi + if [ ! -d "${ROOT}/.home/.yard/gem_index"]; then + ${RUN} bundle exec --gemfile ${ROOT}/Gemfile yard config --gem-install-yri + ${RUN} bundle exec --gemfile ${ROOT}/Gemfile yard gems + touch ${ROOT}/.stamps/dev_gems + fi fi if [ ! -d ${ROOT}/node_modules ]; then - ${RUN} npm i + ${RUN} npm i fi if [ -v DEVCONTAINER_ENV ]; then - BUNDLE="bundle" - RUBY="bundle exec ruby" - RAKE="bundle exec rake" - NPM="npm" - NPX="npx" - NODE="node" - PYTHON="${ROOT}/.home/.venv/bin/python3" - PIP="${ROOT}/.home/.venv/bin/pip" - BASH="bash" + BUNDLE="bundle" + RUBY="bundle exec ruby" + RAKE="bundle exec rake" + NPM="npm" + NPX="npx" + NODE="node" + PYTHON="${ROOT}/.home/.venv/bin/python3" + PIP="${ROOT}/.home/.venv/bin/pip" + BASH="bash" +elif [ "${CONTAINER_TYPE}" == "docker" ]; then + BUNDLE="${DOCKER_BASE} bundle" + RUBY="${DOCKER_BASE} bundle exec ruby" + RAKE="${DOCKER_BASE} bundle exec rake" + NPM="${DOCKER_BASE} npm" + NPX="${DOCKER_BASE} npx" + NODE="${DOCKER_BASE} node" + PYTHON="${DOCKER_BASE} ${ROOT}/.home/.venv/bin/python3" + PIP="${DOCKER_BASE} ${ROOT}/.home/.venv/bin/pip" + BASH="${DOCKER_BASE} bash" else - BUNDLE="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle" - RUBY="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle exec ruby" - RAKE="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle exec rake" - NPM="singularity run ${HOME_OPT} ${CONTAINER_PATH} npm" - NPX="singularity run ${HOME_OPT} ${CONTAINER_PATH} npx" - NODE="singularity run ${HOME_OPT} ${CONTAINER_PATH} node" - PYTHON="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/python3" - PIP="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/pip" - BASH="singularity run ${HOME_OPT} ${CONTAINER_PATH} bash" + BUNDLE="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle" + RUBY="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle exec ruby" + RAKE="singularity run ${HOME_OPT} ${CONTAINER_PATH} bundle exec rake" + NPM="singularity run ${HOME_OPT} ${CONTAINER_PATH} npm" + NPX="singularity run ${HOME_OPT} ${CONTAINER_PATH} npx" + NODE="singularity run ${HOME_OPT} ${CONTAINER_PATH} node" + PYTHON="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/python3" + PIP="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/pip" + BASH="singularity run ${HOME_OPT} ${CONTAINER_PATH} bash" fi if [ ! -f $ROOT/.git/hooks/pre-commit ]; then @@ -125,7 +215,7 @@ if [ -x "\$INSTALL_PYTHON" ]; then elif command -v pre-commit > /dev/null; then exec pre-commit "\${ARGS[@]}" else - echo '`pre-commit` not found. Did you forget to activate your virtualenv?' 1>&2 + echo '\`pre-commit\` not found. Did you forget to activate your virtualenv?' 1>&2 exit 1 fi HOOK