################################################################################
## Copyright (c) 2014-2022, Lawrence Livermore National Security, LLC.
## Produced at the Lawrence Livermore National Laboratory.
## Written by the LBANN Research Team (B. Van Essen, et al.) listed in
## the CONTRIBUTORS file. <lbann-dev@llnl.gov>
##
## LLNL-CODE-697807.
## All rights reserved.
##
## This file is part of LBANN: Livermore Big Artificial Neural Network
## Toolkit. For details, see http://software.llnl.gov/LBANN or
## https://github.com/LLNL/LBANN.
##
## Licensed under the Apache License, Version 2.0 (the "Licensee"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at:
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
## implied. See the License for the specific language governing
## permissions and limitations under the license.
################################################################################

# This is the testing pipeline for the Pascal cluster at LLNL. This
# cluster builds the LBANN applications and libraries using a single
# compiler toolchain and then runs a collection of tests. Testing
# output is in JUnit format and parsed by the pipeline for web
# viewing.

# See the Catalyst pipeline for more thorough documentation.

stages:
  - allocate
  - build
  - test
  - deallocate

# Get LC resources.
allocate lc resources:
  stage: allocate
  extends: .pascal common
  variables:
    GIT_STRATEGY: none
  script:
    - echo "== ACQUIRING SLURM RESOURCES =="
    - salloc --exclusive -N 1 -p pbatch -t 45 --no-shell -J ${JOB_NAME}
  timeout: 6h

# Build LBANN and establish the Spack environment for this pipeline.
build and install:
  extends: .pascal common
  stage: build
  artifacts:
    paths:
      - spack-*.txt
      - spack-build-*/CMakeCache.txt
      - spack-build-*/build.ninja
      - spack-build-*/spack-*.txt
      - spack-ci-env-name.sh
      - ${RESULTS_DIR}/*
  script:
    - echo "== BUILDING LBANN =="
    - export JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - export BUILD_TASKS=$(($(nproc) + 2))
    - echo "SPACK_REPO=${HOME}/${SPACK_REPO}"
    - export GITLAB_SUFFIX=`basename ${CI_BUILDS_DIR}`
    - export SPACK_ENV_NAME=${SPACK_ENV_BASE_NAME}-${GITLAB_SUFFIX}
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - srun --jobid=${JOB_ID} -N 1 -t 30 ./scripts/build_lbann.sh -r
      -l ${SPACK_ENV_NAME} -j ${BUILD_TASKS}
      -e ./scripts/common_spack_packages/ci_spack_packages.sh
      ${BUILD_SCRIPT_OPTIONS} -- 
      +deterministic +vision +numpy +unit_tests ${SPACK_SPECS}
    - export TEST_TASKS_PER_NODE=2
    - export TEST_MPIBIND_FLAG="--mpibind=off"
    - export SPACK_ARCH=$(spack arch)
    - export SPACK_ARCH_TARGET=$(spack arch -t)
    - echo "export SPACK_DEP_ENV_NAME=${SPACK_ENV_NAME}" > spack-ci-env-name.sh
    - echo "export SPACK_ARCH=${SPACK_ARCH}" >> spack-ci-env-name.sh
    - echo "export SPACK_ARCH_TARGET=${SPACK_ARCH_TARGET}" >> spack-ci-env-name.sh
    - .gitlab/common/run-catch-tests.sh

# This is a dummy job that checks the Catch2 testing.
check catch2 tests:
  extends:
    - .pascal common
  stage: test
  dependencies:
    - build and install
  script:
    - ([[ $(find ${RESULTS_DIR} -name "catch-tests-failed.txt" | wc -l) -eq 0 ]])
  artifacts:
    reports:
      junit: ${RESULTS_DIR}/*.xml

# Cleanup the pipeline's Spack environment.
# Switching over to reusing Spack environments for each feature branch so don't remove them immediately
# Cleanup any build directories and spack environments older than 5 days since last use
remove spack environment:
  extends: .pascal common
  stage: deallocate
  variables:
    GIT_STRATEGY: none
  when: always
  script:
    - echo "== REMOVING OLD BUILD RESOURCES =="
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - find ${CI_PROJECT_DIR} -maxdepth 1 -mtime +5 -name "spack-build-*" -type d -exec rm -r {} \;
    - find ${HOME}/${SPACK_REPO}/var/spack/environments/ -maxdepth 1 -mtime +5 -type d -exec basename {} \; | xargs -r spack env rm --yes-to-all

# Free the allocation we obtained in "allocate lc resources".
release allocation:
  stage: deallocate
  extends: .pascal common
  variables:
    GIT_STRATEGY: none
  when: always
  script:
    - echo "== RELEASING RESOURCES =="
    - export JOB_ID=$(squeue -h -n "${JOB_NAME}" -o "%A")
    - ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})

# Load the spack shell integration and load the environment.
.uses spack environment:
  before_script:
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - source spack-ci-env-name.sh
    - spack env activate lbann-${SPACK_ENV_NAME}-${SPACK_ARCH_TARGET}
    - spack load lbann@${SPACK_ENV_NAME}-${SPACK_ARCH_TARGET} arch=${SPACK_ARCH}

# Variables for Pascal.
.pascal common:
  variables:
    # Just the obvious identifier. Which specific node doesn't matter.
    SYSTEM_NAME: pascal
    SPACK_REPO: spack_repos/spack_${SYSTEM_NAME}.git

    # This is based on the assumption that each runner will only ever
    # be able to run one pipeline on a given cluster at one time.
    SPACK_ENV_BASE_NAME: gitlab-${CI_COMMIT_BRANCH}-${GITLAB_USER_LOGIN}-${SYSTEM_NAME}-gcc-${CI_RUNNER_SHORT_TOKEN}

    # These are system-specific specs that should be forwarded to the
    # build script
#    BUILD_SCRIPT_OPTIONS: ""
    
    # These are system-specific specs that should be forwarded to the
    # build script
#    SPACK_SPECS: "+cuda +half +fft"

    # This variable is the name used to identify the job in the Slurm
    # queue. We need this to be able to access the correct jobid.
    JOB_NAME: ${CI_PROJECT_NAME}_${CI_PIPELINE_ID}

    # This is needed to ensure that we run as lbannusr.
    LLNL_SERVICE_USER: lbannusr

    # Catch2 output.
    RESULTS_DIR: results-${CI_PIPELINE_ID}

    # Preserve any spack-build-* directories to allow for faster testing
    GIT_CLEAN_FLAGS: -ffdx -e spack-build-*

  tags:
    - pascal
    - shell
