name: Mars CI on Platforms

on:
  push:
    branches:
      - '*'
  pull_request:
    types: ['opened', 'reopened', 'synchronize']

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: [3.8-kubernetes, 3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
        include:
          - { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
              no-deploy: 1, with-kubernetes: "with Kubernetes" }
          - { os: ubuntu-20.04, python-version: 3.8-hadoop, no-common-tests: 1,
              no-deploy: 1, with-hadoop: "with hadoop" }
          - { os: ubuntu-latest, python-version: 3.8-vineyard, no-common-tests: 1,
              no-deploy: 1, with-vineyard: "with vineyard" }
          - { os: ubuntu-latest, python-version: 3.8-ray, no-common-tests: 1,
              no-deploy: 1, with-ray: "with ray" }
          - { os: ubuntu-latest, python-version: 3.8-ray-deploy, no-common-tests: 1,
              no-deploy: 1, with-ray-deploy: "with ray deploy" }
          - { os: ubuntu-latest, python-version: 3.8-ray-dag, no-common-tests: 1,
              no-deploy: 1, with-ray-dag: "with ray dag" }
          - { os: ubuntu-latest, python-version: 3.8-dask, no-common-tests: 1,
              no-deploy: 1, run-dask: "run dask" }

    steps:
      - name: Check out code
        uses: actions/checkout@v2
        with:
          fetch-depth: 2

      - name: Set up conda ${{ matrix.python-version }}
        env:
          PYTHON: ${{ matrix.python-version }}
        shell: bash
        run: |
          source ./ci/install-conda.sh
          python -m pip install --upgrade pip setuptools wheel coverage

      - name: Start minikube
        if: ${{ matrix.with-kubernetes }}
        uses: medyagh/setup-minikube@master
        with:
          driver: docker
          mount-path: '/home/runner:/home/runner'

      - name: Install dependencies
        env:
          WITH_HADOOP: ${{ matrix.with-hadoop }}
          WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
          WITH_VINEYARD: ${{ matrix.with-vineyard }}
          WITH_RAY: ${{ matrix.with-ray }}
          WITH_RAY_DEPLOY: ${{ matrix.with-ray-deploy }}
          WITH_RAY_DAG: ${{ matrix.with-ray-dag }}
          RUN_DASK: ${{ matrix.run-dask }}
          NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
        shell: bash
        run: |
          source ./ci/reload-env.sh
          export DEFAULT_VENV=$VIRTUAL_ENV

          source ./ci/rewrite-cov-config.sh

          pip install numpy scipy cython

          pip install -e ".[dev,extra]"

          if [[ $UNAME == "windows" ]]; then
            pip install virtualenv flaky
          else
            pip install virtualenv flaky
            if [ -n "$WITH_KUBERNETES" ]; then
              pip install kubernetes "pandas<1.5.0"
              kubectl get pods -A
            fi
            if [ -n "$WITH_HADOOP" ]; then
              ./ci/install-hadoop.sh
              echo "import coverage; coverage.process_startup()" > \
                $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
              conda install -n test --quiet --yes -c conda-forge python=$PYTHON skein libffi conda-pack
            fi
            if [ -n "$WITH_VINEYARD" ]; then
              pip install vineyard -i https://pypi.org/simple

              mkdir -p /tmp/etcd-download-test
              export ETCD_VER=v3.4.13
              export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
              curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
              tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
              sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
              sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
              rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
            fi
            if [ -n "$WITH_RAY" ] || [ -n "$WITH_RAY_DAG" ] || [ -n "$WITH_RAY_DEPLOY" ]; then
              pip install "ray>=1.8.0,<2.4.0"
              pip install "xgboost_ray<0.1.14" "protobuf<4"
              # Ray Datasets need pyarrow>=6.0.1
              pip install "pyarrow>=6.0.1"
              pip install lightgbm
            fi
            if [ -n "$RUN_DASK" ]; then
              pip install "dask[complete]" "mimesis<9.0.0" scikit-learn
            fi
          fi
          conda list -n test

      - name: Test with pytest
        env:
          WITH_HADOOP: ${{ matrix.with-hadoop }}
          WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
          WITH_CYTHON: ${{ matrix.with-cython }}
          WITH_VINEYARD: ${{ matrix.with-vineyard }}
          WITH_RAY: ${{ matrix.with-ray }}
          WITH_RAY_DEPLOY: ${{ matrix.with-ray-deploy }}
          WITH_RAY_DAG: ${{ matrix.with-ray-dag }}
          RUN_DASK: ${{ matrix.run-dask }}
          NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
          NUMPY_EXPERIMENTAL_ARRAY_FUNCTION: 1
          USE_MINIKUBE_DOCKER_ENV: true
          CHANGE_MINIKUBE_NONE_USER: true
        shell: bash
        run: |
          source ./ci/reload-env.sh

          if [ -n "$WITH_HADOOP" ]; then
            source $CONDA/bin/activate test
            source ./ci/reload-env.sh
            pytest $PYTEST_CONFIG -m hadoop
            coverage report
          fi
          if [ -n "$WITH_KUBERNETES" ]; then
            pytest $PYTEST_CONFIG --forked mars/deploy/kubernetes
            coverage report
          fi
          if [ -n "$WITH_VINEYARD" ]; then
            pytest $PYTEST_CONFIG mars/storage/tests/test_libs.py
            mv .coverage build/.coverage.test_lib.file

            pytest $PYTEST_CONFIG mars/deploy/oscar/tests/test_local.py
            mv .coverage build/.coverage.test_local.file

            pytest $PYTEST_CONFIG -k "vineyard" \
                mars/tensor/datastore/tests/test_datastore_execution.py \
                mars/dataframe/datastore/tests/test_datastore_execution.py
            mv .coverage build/.coverage.test_vineyard_op.file

            coverage combine build/ && coverage report
          fi
          if [ -n "$WITH_RAY" ]; then
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s --ignore=mars/deploy/oscar/ -m ray 
            coverage report
          fi
          if [ -n "$WITH_RAY_DEPLOY" ]; then
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray.py -m ray
            mv .coverage build/.coverage.test_ray.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_load_modules.py -m ray
            mv .coverage build/.coverage.test_ray_load_modules.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_cluster_standalone.py -m ray
            mv .coverage build/.coverage.test_ray_cluster_standalone.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_client.py -m ray
            mv .coverage build/.coverage.test_ray_client.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_fault_injection.py -m ray
            mv .coverage build/.coverage.test_ray_fault_injection.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_scheduling.py -m ray
            mv .coverage build/.coverage.test_ray_scheduling.file
          
            coverage combine build/ && coverage report
          fi
          if [ -n "$WITH_RAY_DAG" ]; then
            export MARS_CI_BACKEND=ray
            export RAY_idle_worker_killing_time_threshold_ms=60000
            pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/dataframe -v -s -m "not skip_ray_dag" --ignore=mars/dataframe/contrib/raydataset
            pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/dataframe/contrib/raydataset -v -s -m "not skip_ray_dag" 
            pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/tensor -v -s -m "not skip_ray_dag"
            pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/learn --ignore mars/learn/contrib --ignore mars/learn/utils/tests/test_collect_ports.py -v -s -m "not skip_ray_dag"
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s -m ray_dag
            mv .coverage build/.coverage.ray_dag.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag.py
            mv .coverage build/.coverage.test_ray_dag.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag_failover.py
            mv .coverage build/.coverage.test_ray_dag_failover.file
            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag_oscar.py -m ray
            mv .coverage build/.coverage.test_ray_dag_oscar.file
            
            coverage combine build/ && coverage report
          fi
          if [ -n "$RUN_DASK" ]; then
            pytest $PYTEST_CONFIG mars/contrib/dask/tests/test_dask.py
            coverage report
          fi
          coverage xml

      - name: Stop vineyard runtime
        if: ${{ matrix.with-vineyard }}
        shell: bash
        run: |
          sudo docker stop vineyard || true

      - name: Report coverage data
        shell: bash
        run: |
          bash <(curl -s https://codecov.io/bash)
