[WIP] Adding Stain Normalization Transform for Pathology

.dockerignore

@@ -8,7 +8,6 @@ docs/
 .coverage/
 coverage.xml
 .readthedocs.yml
-*.md
 *.toml
 
 !README.md

.github/workflows/cron.yml

@@ -3,6 +3,8 @@ name: crons
 on:
   schedule:
     - cron: "0 2 * * *"  # at 02:00 UTC
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   cron-gpu:
@@ -40,17 +42,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
-        stop_time=$((LAUNCH_DELAY + $(date +%s)))
-        while [ $(date +%s) -lt $stop_time ]; do
-          python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))';
-        done
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
-        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
-        echo $CUDA_VISIBLE_DEVICES
         BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
@@ -83,17 +82,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
-        stop_time=$((LAUNCH_DELAY + $(date +%s)))
-        while [ $(date +%s) -lt $stop_time ]; do
-          python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))';
-        done
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
-        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
-        echo $CUDA_VISIBLE_DEVICES
         BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
@@ -115,14 +111,15 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
         BUILD_MONAI=1 ./runtests.sh --coverage --pytype --unittests  # unit tests with pytype checks, coverage report
-        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
-        echo $CUDA_VISIBLE_DEVICES
         BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
@@ -156,8 +153,12 @@ jobs:
         cd tutorials
         python -m pip install -r requirements.txt
     - name: Run tutorial notebooks
+      timeout-minutes: 150
       run: |
         export CUDA_VISIBLE_DEVICES=${{ steps.monai-install.outputs.devices }}
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         cd /opt/tutorials
         $(pwd)/runner.sh
+        if pgrep python; then pkill python; fi

.github/workflows/docker.yml

@@ -0,0 +1,123 @@
+name: docker
+# versioning: compute a static version file
+# local_docker: use the version file to build docker images
+# docker_test_latest: test the latest internal docker image (has flake)
+# docker_test_dockerhub: test the latest dockerhub release (no flake)
+on:
+  # master only docker deployment and quick tests
+  push:
+    branches:
+      - master
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  versioning:
+    # compute versioning file from python setup.py
+    # upload as artifact
+    # (also used in release.yml)
+    if: github.repository == 'Project-MONAI/MONAI'
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        # full history so that we can git describe
+        with:
+          ref: master
+          fetch-depth: 0
+      - shell: bash
+        run: |
+          git describe
+          python setup.py build
+          cat build/lib/monai/_version.py
+      - name: Upload version
+        uses: actions/upload-artifact@v2
+        with:
+          name: _version.py
+          path: build/lib/monai/_version.py
+      - name: Clean up directory
+        shell: bash
+        run: |
+          ls -al
+          rm -rf {*,.[^.]*}
+
+  local_docker:
+    # builds two versions: local_monai:latest and local_monai:dockerhub
+    # latest: used for local tests
+    # dockerhub: release, no flake package
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: versioning
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: master
+    - name: Download version
+      uses: actions/download-artifact@v2
+      with:
+        name: _version.py
+    - name: docker_build
+      shell: bash
+      run: |
+        # get tag info for versioning
+        cat _version.py
+        mv _version.py monai/
+        # build and run original docker image for local registry
+        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
+        docker push localhost:5000/local_monai:latest
+        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
+        sed -i '/flake/d' requirements-dev.txt
+        docker build -t projectmonai/monai:latest -f Dockerfile .
+        # also push as tag "dockerhub" to local registry
+        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
+        docker push localhost:5000/local_monai:dockerhub
+        # distribute as always w/ tag "latest" to hub.docker.com
+        echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
+        docker push projectmonai/monai:latest
+        docker logout
+        docker image prune -f
+
+  docker_test_latest:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+        if pgrep python; then pkill python; fi
+      env:
+        QUICKTEST: True
+
+  docker_test_dockerhub:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:dockerhub
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+        if pgrep python; then pkill python; fi
+      env:
+        QUICKTEST: True

.github/workflows/integration.yml

@@ -42,9 +42,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --unittests --net
+        BUILD_MONAI=1 ./runtests.sh --net
+        BUILD_MONAI=1 ./runtests.sh --unittests
+        if pgrep python; then pkill python; fi
+      shell: bash
     - name: Add reaction
       uses: peter-evans/create-or-update-comment@v1
       with:

.github/workflows/pythonapp.yml

@@ -41,7 +41,7 @@ jobs:
         # Git hub actions have 2 cores, so parallize pytype
         $(pwd)/runtests.sh --codeformat -j 2
 
-  quick-py3:  # full dependencies installed
+  quick-py3:  # full dependencies installed tests for different OS
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -105,7 +105,7 @@ jobs:
       env:
         QUICKTEST: True
 
-  min-dep-py3:  # min dependencies installed
+  min-dep-os:  # min dependencies installed tests for different OS
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -154,6 +154,51 @@ jobs:
       env:
         QUICKTEST: True
 
+  min-dep-py3:  # min dependencies installed tests for different python
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7]
+    timeout-minutes: 40
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Prepare pip wheel
+      run: |
+        which python
+        python -m pip install --user --upgrade pip setuptools wheel
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "::set-output name=datew::$(date '+%Y-%V')"
+        echo "::set-output name=dir::$(pip cache dir)"
+      shell: bash
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ubuntu-latest-latest-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install the dependencies
+      run: |
+        # min. requirements
+        python -m pip install torch==1.8.1
+        python -m pip install -r requirements-min.txt
+        python -m pip list
+        BUILD_MONAI=0 python setup.py develop  # no compile of extensions
+      shell: bash
+    - name: Run quick tests (CPU ${{ runner.os }})
+      run: |
+        python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
+        python -c "import monai; monai.config.print_config()"
+        python -m tests.min_tests
+      env:
+        QUICKTEST: True
+
   GPU-quick-py3:  # GPU with full dependencies
     if: github.repository == 'Project-MONAI/MONAI'
     strategy:
@@ -236,21 +281,28 @@ jobs:
         python -m pip install --upgrade pip wheel
         python -m pip install ${{ matrix.pytorch }}
         python -m pip install -r requirements-dev.txt
+        python -m pip list
     - name: Run quick tests (GPU)
       run: |
-        python -m pip list
         nvidia-smi
+        export LAUNCH_DELAY=$(python -c "import numpy; print(numpy.random.randint(30) * 5)")
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
         export CUDA_VISIBLE_DEVICES=$(coverage run -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
         BUILD_MONAI=1 ./runtests.sh --quick --unittests
-        if [ ${{ matrix.environment }} == "PT18+CUDA112" ]; then
+        if [ ${{ matrix.environment }} = "PT18+CUDA112" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
         fi
         coverage xml
+        if pgrep python; then pkill python; fi
+      shell: bash
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with: