################################################################################
## Copyright (c) 2014-2022, Lawrence Livermore National Security, LLC.
## Produced at the Lawrence Livermore National Laboratory.
## Written by the LBANN Research Team (B. Van Essen, et al.) listed in
## the CONTRIBUTORS file. <lbann-dev@llnl.gov>
##
## LLNL-CODE-697807.
## All rights reserved.
##
## This file is part of LBANN: Livermore Big Artificial Neural Network
## Toolkit. For details, see http://software.llnl.gov/LBANN or
## https://github.com/LLNL/LBANN.
##
## Licensed under the Apache License, Version 2.0 (the "Licensee"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at:
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
## implied. See the License for the specific language governing
## permissions and limitations under the license.
################################################################################

# This is the testing pipeline for the Catalyst cluster at LLNL. This
# cluster builds the LBANN applications and libraries using a single
# compiler toolchain and then runs a collection of tests. Testing
# output is in JUnit format and parsed by the pipeline for web
# viewing.

stages:
  - allocate
  - build
  - test
  - deallocate

# The purpose of this job is simply to allocate some nodes for the
# pipeline to utilize. The trick here is JOB_NAME. Later on, we will
# extract the job ID from the SLURM queue using this name. It will be
# unique to this pipeline and therefore shared across all jobs in this
# pipeline.
allocate lc resources:
  stage: allocate
  extends: .catalyst common
  variables:
    GIT_STRATEGY: none
  script:
    - echo "== ACQUIRING SLURM RESOURCES =="
    - echo "${WITH_WEEKLY:+Running with --weekly}"
    - export TEST_TIME=$([[ -n "${WITH_WEEKLY}" ]] && echo "120" || echo "90")
    - export LBANN_NNODES=$([[ -n "${WITH_WEEKLY}" ]] && echo "4" || echo "2")
    - salloc --exclusive -N ${LBANN_NNODES} -p pbatch -t ${TEST_TIME} --no-shell -J ${JOB_NAME}
  timeout: 6h

# This replaces "test_compiler.py". This has the advantage that it
# very explicitly approximates what a user would do to build LBANN.
#
# This job also establishes the Spack environment that we will
# use. We'll use the same environment for the whole pipeline. This
# should be fine as the test jobs will view the environment as
# "read-only".
build and install:
  extends: .catalyst common
  stage: build
  artifacts:
    paths:
      - spack-*.txt
      - spack-build-*/CMakeCache.txt
      - spack-build-*/build.ninja
      - spack-build-*/spack-*.txt
      - spack-ci-env-name.sh
      - ${RESULTS_DIR}/*
  script:
    - echo "== BUILDING LBANN =="
    - export JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - export BUILD_TASKS=$(($(nproc) + 2))
    - export GITLAB_SUFFIX=`basename ${CI_BUILDS_DIR}`
    - export SPACK_ENV_NAME=${SPACK_ENV_BASE_NAME}-${GITLAB_SUFFIX}
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - srun --jobid=${JOB_ID} -N 1 -t 30 ./scripts/build_lbann.sh -r
      -l ${SPACK_ENV_NAME} -j ${BUILD_TASKS}
      -p py-pip --ci-pip --
      +deterministic +vision +numpy +unit_tests ${SPACK_SPECS}
    - export TEST_TASKS_PER_NODE=4
    - export TEST_MPIBIND_FLAG="--mpibind=off"
    - export SPACK_ARCH=$(spack arch)
    - export SPACK_ARCH_TARGET=$(spack arch -t)
    - echo "export SPACK_DEP_ENV_NAME=${SPACK_ENV_NAME}" > spack-ci-env-name.sh
    - echo "export SPACK_ARCH=${SPACK_ARCH}" >> spack-ci-env-name.sh
    - echo "export SPACK_ARCH_TARGET=${SPACK_ARCH_TARGET}" >> spack-ci-env-name.sh
    - .gitlab/common/run-catch-tests.sh

# The next two jobs effectively replace the rest of "run.sh".
#
# There is a bit of a hack going on here: instead of passing
# "--jobid=${JOB_ID}" to an srun command, we "export
# SLURM_JOB_ID=${JOB_ID}" into the environment prior to launching the
# test script. The reason for this is that the srun commands are
# buried deep inside the Python infrastructure and it would be
# nontrivial to push them all the way in (not so nontrivial that this
# wouldn't be worth doing, but nontrivial enough that it should be its
# own PR).
#
# This appears to be working with one main side effect: as written,
# the integration tests and the unit tests will interfere with each
# other. It's not clear if they are interleaving or overlapping. This
# happens because the two jobs in the test stage will run
# concurrently. We can change this if it's shown to be a problem.
unit tests:
  extends:
    - .catalyst common
    - .uses spack environment
  stage: test
  dependencies:
    - build and install
  script:
    - echo "== RUNNING PYTHON-BASED UNIT TESTS =="
    - echo "Testing $(which lbann)"
    - export OMP_NUM_THREADS=10
    - export SLURM_JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - cd ci_test/unit_tests
    - python3 -m pytest -s -vv --durations=0 --junitxml=results.xml
  artifacts:
    when: always
    paths:
      - ci_test/unit_tests/results.xml
    reports:
      junit: ci_test/unit_tests/results.xml

integration tests:
  extends:
    - .catalyst common
    - .uses spack environment
  stage: test
  dependencies:
    - build and install
  script:
    - echo "== RUNNING PYTHON-BASED INTEGRATION TESTS =="
    - echo "Testing $(which lbann)"
    - export OMP_NUM_THREADS=10
    - export SLURM_JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - cd ci_test/integration_tests
    - export WEEKLY_FLAG=${WITH_WEEKLY:+--weekly}
    - echo "python3 -m pytest -s -vv --durations=0 ${WEEKLY_FLAG} --junitxml=results.xml"
    - python3 -m pytest -s -vv --durations=0 ${WEEKLY_FLAG} --junitxml=results.xml
  artifacts:
    when: always
    paths:
      - ci_test/integration_tests/results.xml
    reports:
      junit: ci_test/integration_tests/results.xml

# This is a dummy job that checks the output of the Catch2
# testing. This is one of the gotchas of GitLab: The Catch2 test
# executables are artifacts of the build procedure, but we don't save
# the entire build as artifacts (it would be very large and there's no
# need since the rest of the tests run on the installed targets (for
# better or worse)). For maximum clarity from the web interface, we
# write a file when the Catch tests fail but then return success from
# the script that runs them. This means that the build job will fail
# iff the build fails. This job will be marked as "failed" if any of
# the catch tests failed, and the pytest-based unit/integration tests
# will run any time the build succeeds.
check catch2 tests:
  extends:
    - .catalyst common
  stage: test
  dependencies:
    - build and install
  script:
    - ([[ $(find ${RESULTS_DIR} -name "catch-tests-failed.txt" | wc -l) -eq 0 ]])
  artifacts:
    reports:
      junit: ${RESULTS_DIR}/*.xml

# Cleanup this pipeline's spack environment.
# Switching over to reusing Spack environments for each feature branch so don't remove them immediately
# Cleanup any build directories and spack environments older than 5 days since last use
remove spack environment:
  extends: .catalyst common
  stage: deallocate
  variables:
    GIT_STRATEGY: none
  when: always
  script:
    - echo "== REMOVING OLD BUILD RESOURCES =="
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - find ${CI_PROJECT_DIR} -maxdepth 1 -mtime +5 -name "spack-build-*" -type d -exec rm -r {} \;
    - find ${HOME}/${SPACK_REPO}/var/spack/environments/ -maxdepth 1 -mtime +5 -type d -exec basename {} \; | xargs -r spack env rm --yes-to-all

# This frees the allocation we obtained in "allocate lc resources".
release allocation:
  stage: deallocate
  extends: .catalyst common
  variables:
    GIT_STRATEGY: none
  when: always
  script:
    - echo "== RELEASING RESOURCES =="
    - export JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})

# This loads the spack shell integration and then loads the
# environment. Since this relies on the environment for this pipeline
# existing, it should only be used after the build stage (i.e., in
# testing).
.uses spack environment:
  before_script:
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - source spack-ci-env-name.sh
    - spack env activate lbann-${SPACK_DEP_ENV_NAME}-${SPACK_ARCH_TARGET}
    - spack load lbann@${SPACK_DEP_ENV_NAME}-${SPACK_ARCH_TARGET} arch=${SPACK_ARCH}

# For simplicity, I have put the variables as well as the tags
# here. The variables could just be top-level in the file (perhaps the
# tags could be, too, but I'm not sure), but this makes extracting
# them to another file easier down the road.
.catalyst common:
  variables:
    # Just the obvious identifier. Which specific node doesn't matter.
    SYSTEM_NAME: catalyst
    SPACK_REPO: spack_repos/spack_${SYSTEM_NAME}.git

    # This is based on the assumption that each runner will only ever
    # be able to run one pipeline on a given cluster at one time.
    SPACK_ENV_BASE_NAME: gitlab-${CI_COMMIT_BRANCH}-${GITLAB_USER_LOGIN}-${SYSTEM_NAME}-${CI_RUNNER_SHORT_TOKEN}

    # These are system-specific specs that should be forwarded to the
    # build script
    SPACK_SPECS: "+onednn +half +fft"

    # This variable is the name used to identify the job in the Slurm
    # queue. We need this to be able to access the correct jobid.
    JOB_NAME: ${CI_PROJECT_NAME}_${CI_PIPELINE_ID}

    # This is needed to ensure that we run as lbannusr.
    LLNL_SERVICE_USER: lbannusr

    # This directory is a pipeline-unique dumping ground for any
    # output files that would be hard to track down later. (In
    # particular, I ran a few tests where some stages had multiple
    # "spack-build-*" directories, so they were reporting dozens of
    # xml files of Catch2 output (there should be 17 -- 1 seq, 8 mpi,
    # 8 mpi-fs). Since the "*" there is a hash that's not computable
    # in this file, we cannot specify it correctly in an "artifacts"
    # section. So we just create our own results cache with known
    # name.
    #
    # NOTE: directories specified in a pipeline file are relative to
    # ${CI_PROJECT_DIR}. Therefore, shell scripts should be sure of
    # their working directory before attempting to use this as a
    # relative path. Alternatively, they should prefix
    # ${CI_PROJECT_DIR} and use absolute paths (absolute paths are not
    # allowed in pipeline files either, which is why this is not an
    # absolute path).
    RESULTS_DIR: results-${CI_PIPELINE_ID}

    # This needs to be imported here, too. Failure to do so causes
    # problems if it's not set.
    LBANN_CI_RUN_WEEKLY: ${LBANN_CI_RUN_WEEKLY}

    # Preserve any spack-build-* directories to allow for faster testing
    GIT_CLEAN_FLAGS: -ffdx -e spack-build-*

  tags:
    - catalyst
    - shell
