diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..2dda5ec --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,342 @@ +version: 2 +jobs: + get_data: + machine: + image: circleci/classic:201711-01 + working_directory: /tmp/data + steps: + - restore_cache: + keys: + - data-v3-{{ .Revision }} + - data-v3- + - run: + name: Get ds114_test1 + command: | + mkdir -p /tmp/data + if [[ ! -d /tmp/data/ds114_test1 ]]; then + wget -c -O ~/ds114_test1.tar \ + "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e54a326c613b01d7d3ed90" && \ + tar xf ~/ds114_test1.tar -C /tmp/data && \ + rm ~/ds114_test1.tar; + else + echo "ds114_test1 was cached" + fi + - run: + name: Get ds114_test2 + command: | + mkdir -p /tmp/data + if [[ ! -d /tmp/data/ds114_test2 ]]; then + wget -c -O ~/ds114_test2.tar \ + "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e549f9b83f6901d457d162" && \ + tar xf ~/ds114_test2.tar -C /tmp/data && \ + rm ~/ds114_test2.tar; + else + echo "ds114_test2 was cached" + fi + - run: + name: Get ds114_test1_freesurfer + command: | + mkdir -p /tmp/data + if [[ ! -d /tmp/data/ds114_test1_freesurfer ]]; then + wget -c -O ~/ds114_test1_freesurfer.tar \ + "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882adf3b83f6901f564da49" && \ + tar xf ~/ds114_test1_freesurfer.tar -C /tmp/data && \ + rm ~/ds114_test1_freesurfer.tar; + else + echo "ds114_test1_freesurfer was cached" + fi + - run: + name: Get ds114_test2_freesurfer + command: | + mkdir -p /tmp/data + if [[ ! -d /tmp/data/ds114_test2_freesurfer ]]; then + wget -c -O ~/ds114_test2_freesurfer.tar \ + "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882b0e3b83f6901fb64da18" && \ + tar xf ~/ds114_test2_freesurfer.tar -C /tmp/data && \ + rm ~/ds114_test2_freesurfer.tar; + else + echo "ds114_test2_freesurfer was cached" + fi + - save_cache: + key: data-v3-{{ .Revision }}-{{ epoch }} + paths: + - /tmp/data + + build: + docker: + - image: docker:18.01.0-ce-git + steps: + - checkout + - run: + name: Install parallel gzip + command: | + apk add --no-cache pigz + - restore_cache: + keys: + - docker-v2-{{ .Branch }}-{{ .Revision }} + - docker-v2-{{ .Branch }}- + - docker-v2-master- + - docker-v2- + paths: + - /tmp/cache/docker.tar.gz + - setup_remote_docker + - run: + name: Load Docker image layer cache + no_output_timeout: 30m + command: | + docker info + set +o pipefail + if [ -f /tmp/cache/docker.tar.gz ]; then + pigz -d --stdout /tmp/cache/docker.tar.gz | docker load + else + echo "No docker cache found" + fi + docker images + - run: + name: get version + command: | + git describe --tags --always > version + cat version + - run: + name: Build Docker image + no_output_timeout: 60m + command: | + docker images + set +o pipefail + docker build --cache-from=bids/${CIRCLE_PROJECT_REPONAME} \ + -t bids/${CIRCLE_PROJECT_REPONAME}:latest . + docker images + - run: + name: Docker save + no_output_timeout: 40m + command: | + mkdir -p /tmp/cache + set +o pipefail + docker save bids/${CIRCLE_PROJECT_REPONAME}:latest \ + | pigz -2 -p 3 > /tmp/cache/docker.tar.gz + - save_cache: + key: docker-v2-{{ .Branch }}-{{ epoch }} + paths: + - /tmp/cache/docker.tar.gz + # Usage and version + - run: + name: Usage BIDS + command: docker run -ti --rm bids/${CIRCLE_PROJECT_REPONAME}:latest -h + - run: + name: Usage files + command: docker run --rm -ti --entrypoint=run_brain_age_files.py bids/${CIRCLE_PROJECT_REPONAME}:latest -h + - run: + name: Version BIDS + command: docker run -ti --rm bids/${CIRCLE_PROJECT_REPONAME}:latest -v + - run: + name: Version files + command: docker run --rm -ti --entrypoint=run_brain_age_files.py bids/${CIRCLE_PROJECT_REPONAME}:latest -v + - run: + name: Python packages + command: docker run --rm -ti --entrypoint=/bin/bash bids/${CIRCLE_PROJECT_REPONAME}:latest -c "pip freeze" + + test: + machine: + image: circleci/classic:201711-01 + working_directory: /tmp/data + environment: + - DATADIR: '/tmp/data' + - OUTDIR: '/tmp/outputs' + steps: + - run: + name: Install parallel gzip + command: | + sudo apt-get update && sudo apt-get install -y pigz + - restore_cache: + keys: + - docker-v2-{{ .Branch }}-{{ .Revision }} + - docker-v2-{{ .Branch }}- + - docker-v2-master- + - docker-v2- + paths: + - /tmp/cache/docker.tar.gz + - restore_cache: + keys: + - data-v3-{{ .Revision }} + - data-v3- + + # Load Docker + - run: + name: Load Docker + no_output_timeout: 30m + command: | + docker info + set +o pipefail + pigz -d --stdout /tmp/cache/docker.tar.gz | docker load + docker images + + - run: + name: make output directory + command: mkdir -p ${OUTDIR} + + # Test 1: ds114_test1 + - run: + name: Test 1 ds114_test1 - 1 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \ + -v ${DATADIR}/ds114_test1_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test1:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out participant --freesurfer_dir /data/fs \ + --license_key="~/test.key" + - run: + name: Test 1 ds114_test1 - 2 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \ + -v ${DATADIR}/ds114_test1_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test1:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out group --freesurfer_dir /data/fs \ + --license_key="~/test.key" + - run: + name: Test 1 ds114_test1 - 3 + command: | + cat ${OUTDIR}/ds114_test1/baracus/00_group/group_predicted_age.tsv + + + # Test2 : models + - run: + name: Test 2 models - 1 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \ + -v ${DATADIR}/ds114_test1_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test1_models:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out participant --freesurfer_dir /data/fs \ + --license_key="~/test.key" \ + --models Liem2016__OCI_norm Liem2016__full_2samp_training + - run: + name: Test 2 models - 2 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \ + -v ${DATADIR}/ds114_test1_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test1_models:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out group --freesurfer_dir /data/fs \ + --license_key="~/test.key" + - run: + name: Test 2 models - 3 + command: | + cat ${OUTDIR}/ds114_test1_models/baracus/00_group/group_predicted_age.tsv + + # Test3: file mode + - run: + name: Test 3 file mode - 1 + command: | + docker run --rm -ti \ + -v ${OUTDIR}/ds114_test1/baracus/sub-01/data/:/data/in \ + -v ${OUTDIR}/ds114_test1_file_mode:/data/out \ + --entrypoint=run_brain_age_files.py \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/out \ + --lh_thickness_file /data/in/lh.thickness.mgh \ + --rh_thickness_file /data/in/rh.thickness.mgh \ + --lh_area_file /data/in/lh.area.mgh \ + --rh_area_file /data/in/rh.area.mgh \ + --aseg_file /data/in/aseg --participant_label sub-01 + - run: + name: Test 3 file mode - 2 + command: | + cat ${OUTDIR}/ds114_test1_file_mode/sub-01/sub-01_predicted_age.tsv + + + # Test 4: ds114_test2 + - run: + name: Test 1 ds114_test2 - 1 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test2:/data/in \ + -v ${DATADIR}/ds114_test2_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test2:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out participant --freesurfer_dir /data/fs \ + --license_key="~/test.key" + - run: + name: Test 1 ds114_test2 - 2 + command: | + docker run --rm -ti -v ${DATADIR}/ds114_test2:/data/in \ + -v ${DATADIR}/ds114_test2_freesurfer:/data/fs \ + -v ${OUTDIR}/ds114_test2:/data/out \ + bids/${CIRCLE_PROJECT_REPONAME}:latest \ + /data/in /data/out group --freesurfer_dir /data/fs \ + --license_key="~/test.key" + - run: + name: Test 1 ds114_test2 - 3 + command: | + cat ${OUTDIR}/ds114_test2/baracus/00_group/group_predicted_age.tsv + + - store_artifacts: + path: /tmp/outputs/ + + deploy: + machine: + image: circleci/classic:201711-01 + steps: + - run: + name: Install parallel gzip + command: | + sudo apt-get update && sudo apt-get install -y pigz + - restore_cache: + keys: + - docker-v2-{{ .Branch }}-{{ .Revision }} + - docker-v2-{{ .Branch }}- + - docker-v2-master- + - docker-v2- + paths: + - /tmp/cache/docker.tar.gz + + # Load Docker + - run: + name: Load Docker + no_output_timeout: 30m + command: | + docker info + set +o pipefail + pigz -d --stdout /tmp/cache/docker.tar.gz | docker load + docker images + - run: + name: Deploy to Docker Hub + no_output_timeout: 40m + command: | + if [[ -n "$DOCKER_PASS" ]]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag bids/${CIRCLE_PROJECT_REPONAME} bids/${CIRCLE_PROJECT_REPONAME}:unstable + docker push bids/${CIRCLE_PROJECT_REPONAME}:unstable + if [[ -n "$CIRCLE_TAG" ]]; then + docker push bids/${CIRCLE_PROJECT_REPONAME}:latest + docker tag bids/${CIRCLE_PROJECT_REPONAME} bids/${CIRCLE_PROJECT_REPONAME}:$CIRCLE_TAG + docker push bids/${CIRCLE_PROJECT_REPONAME}:$CIRCLE_TAG + fi + fi + +workflows: + version: 2 + build_test_deploy: + jobs: + - get_data: + filters: + tags: + only: /.*/ + - build: + filters: + tags: + only: /.*/ + - test: + requires: + - get_data + - build + filters: + tags: + only: /.*/ + - deploy: + requires: + - test + filters: + branches: + only: master + tags: + only: /.*/ diff --git a/Dockerfile b/Dockerfile index 6f1b916..53d6a28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,10 +77,10 @@ ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 -RUN pip install nibabel -RUN pip install pybids -RUN pip install duecredit - +RUN pip install nibabel==2.2.0 +RUN pip install pybids==0.3 grabbit==0.0.8 +RUN pip install duecredit==0.6.3 +RUN pip install scikit-learn==0.17.1 COPY . /code/ RUN cd /code && ls && pip install -e . diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 0c084cf..0000000 --- a/circle.yml +++ /dev/null @@ -1,60 +0,0 @@ -general: - artifacts: - - "~/outputs" - -machine: - services: - - docker #don't use 1.10 - caching is broken - -dependencies: - cache_directories: - - "~/docker" - - "~/data" - - override: - - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - - git describe --tags > version - - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : - timeout: 21600 - - mkdir -p ~/docker; docker save "bids/${CIRCLE_PROJECT_REPONAME}" > ~/docker/image.tar - - - if [[ ! -d ~/data/ds114_test1 ]]; then wget -c -O ${HOME}/ds114_test1.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e54a326c613b01d7d3ed90" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1.tar -C ${HOME}/data; fi - - if [[ ! -d ~/data/ds114_test2 ]]; then wget -c -O ${HOME}/ds114_test2.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e549f9b83f6901d457d162" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2.tar -C ${HOME}/data; fi - - if [[ ! -d ~/data/ds114_test1_freesurfer ]]; then wget -c -O ${HOME}/ds114_test1_freesurfer.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882adf3b83f6901f564da49" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1_freesurfer.tar -C ${HOME}/data; fi - - if [[ ! -d ~/data/ds114_test2_freesurfer ]]; then wget -c -O ${HOME}/ds114_test2_freesurfer.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882b0e3b83f6901fb64da18" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2_freesurfer.tar -C ${HOME}/data; fi - -test: - override: - - docker run -ti --rm bids/${CIRCLE_PROJECT_REPONAME,,} -h - - docker run -ti --rm bids/${CIRCLE_PROJECT_REPONAME,,} -v - - docker run --rm -ti --entrypoint=run_brain_age_files.py bids/baracus -h - - docker run --rm -ti --entrypoint=run_brain_age_files.py bids/baracus -v - - # ds114_test1 - - docker run --rm -ti -v ${HOME}/data/ds114_test1:/data/in -v ${HOME}/data/ds114_test1_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test1:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out participant --freesurfer_dir /data/fs --license_key="~/test.key" - - docker run --rm -ti -v ${HOME}/data/ds114_test1:/data/in -v ${HOME}/data/ds114_test1_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test1:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out group --freesurfer_dir /data/fs --license_key="~/test.key" - - cat ${HOME}/outputs/ds114_test1/baracus/00_group/group_predicted_age.tsv - - # test models - - docker run --rm -ti -v ${HOME}/data/ds114_test1:/data/in -v ${HOME}/data/ds114_test1_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test1_models:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out participant --freesurfer_dir /data/fs --models Liem2016__OCI_norm Liem2016__full_2samp_training --license_key="~/test.key" - - docker run --rm -ti -v ${HOME}/data/ds114_test1:/data/in -v ${HOME}/data/ds114_test1_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test1_models:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out group --freesurfer_dir /data/fs --license_key="~/test.key" - - cat ${HOME}/outputs/ds114_test1_models/baracus/00_group/group_predicted_age.tsv - - # test file mode - - docker run --rm -ti -v ${HOME}/outputs/ds114_test1/baracus/sub-01/data/:/data/in -v ${HOME}/outputs/ds114_test1_file_mode:/data/out --entrypoint=run_brain_age_files.py bids/baracus /data/out --lh_thickness_file /data/in/lh.thickness.mgh --rh_thickness_file /data/in/rh.thickness.mgh --lh_area_file /data/in/lh.area.mgh --rh_area_file /data/in/rh.area.mgh --aseg_file /data/in/aseg --participant_label sub-01 - - cat ${HOME}/outputs/ds114_test1_file_mode/sub-01/sub-01_predicted_age.tsv - - # ds114_test2 - - docker run --rm -ti -v ${HOME}/data/ds114_test2:/data/in -v ${HOME}/data/ds114_test2_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test2:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out participant --freesurfer_dir /data/fs --license_key="~/test.key" - - docker run --rm -ti -v ${HOME}/data/ds114_test2:/data/in -v ${HOME}/data/ds114_test2_freesurfer:/data/fs -v ${HOME}/outputs/ds114_test2:/data/out bids/${CIRCLE_PROJECT_REPONAME,,} /data/in /data/out group --freesurfer_dir /data/fs --license_key="~/test.key" - - cat ${HOME}/outputs/ds114_test2/baracus/00_group/group_predicted_age.tsv - -deployment: - hub: - owner: BIDS-Apps - tag: /.*/ - commands: - - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:latest; fi : - timeout: 21600 - - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag bids/${CIRCLE_PROJECT_REPONAME,,} bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG; fi : - timeout: 21600