################################################################################
## Copyright (c) 2014-2022, Lawrence Livermore National Security, LLC.
## Produced at the Lawrence Livermore National Laboratory.
## Written by the LBANN Research Team (B. Van Essen, et al.) listed in
## the CONTRIBUTORS file. <lbann-dev@llnl.gov>
##
## LLNL-CODE-697807.
## All rights reserved.
##
## This file is part of LBANN: Livermore Big Artificial Neural Network
## Toolkit. For details, see http://software.llnl.gov/LBANN or
## https://github.com/LLNL/LBANN.
##
## Licensed under the Apache License, Version 2.0 (the "Licensee"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at:
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
## implied. See the License for the specific language governing
## permissions and limitations under the license.
################################################################################

# This is the testing pipeline for the Lassen cluster at LLNL. This
# cluster builds the LBANN applications and libraries using a single
# compiler toolchain and then runs a collection of tests. Testing
# output is in JUnit format and parsed by the pipeline for web
# viewing.

stages:
  - everything
  - cleanup

# In contrast to the Catalyst pipeline, this will look much
# simpler. There is only one functional stage and there's no fiddling
# around with persistent allocations or anything like that. Plus, it
# reuses some of the bamboo infrastructure (run.sh). On the downside,
# it made me look at run.sh and remind me how badly we need to
# refactor that file. So a little up, a little down.

build and test everything:
  extends: .lassen common
  stage: everything
  script:
    - echo "== BUILDING AND TESTING LBANN =="
    - echo "${WITH_WEEKLY:+Running with --weekly}"
    - echo "SPACK_REPO=${HOME}/${SPACK_REPO}"
    - export GITLAB_SUFFIX=`basename ${CI_BUILDS_DIR}`
    - export SPACK_ENV_NAME=${SPACK_ENV_BASE_NAME}-${GITLAB_SUFFIX}
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - export TEST_TIME=$([[ -n "${WITH_WEEKLY}" ]] && echo "120" || echo "90")
    - export LBANN_NNODES=$([[ -n "${WITH_WEEKLY}" ]] && echo "4" || echo "2")
    - export WEEKLY_FLAG=${WITH_WEEKLY:+--weekly}
    - lalloc ${LBANN_NNODES} -W ${TEST_TIME} -q pbatch ci_test/run.sh ${WEEKLY_FLAG}
  artifacts:
    when: always
    paths:
      - spack-*.txt
      - spack-build-*/CMakeCache.txt
      - spack-build-*/build.ninja
      - spack-build-*/spack-*.txt
      - ci_test/**/*.xml
      - ci_test/compiler_tests/**/*.txt
    reports:
      junit: ci_test/**/*.xml
  timeout: 6h

# Cleanup this pipeline's spack environment.
# Switching over to reusing Spack environments for each feature branch so don't remove them immediately
# Cleanup any build directories and spack environments older than 5 days since last use
remove spack environment:
  extends: .lassen common
  stage: cleanup
  variables:
    GIT_STRATEGY: none
  when: always
  script:
    - echo "== REMOVING OLD BUILD RESOURCES =="
    - source ${HOME}/${SPACK_REPO}/share/spack/setup-env.sh
    - find ${CI_PROJECT_DIR} -maxdepth 1 -mtime +5 -name "spack-build-*" -type d -exec rm -r {} \;
    - find ${HOME}/${SPACK_REPO}/var/spack/environments/ -maxdepth 1 -mtime +5 -type d -exec basename {} \; | xargs -r spack env rm --yes-to-all

# Variables for Lassen. Because this test uses "test_compiler.py", we
# don't need to specify the Spack spec or the job name.
.lassen common:
  variables:
    # Just the obvious identifier. Which specific node doesn't matter.
    SYSTEM_NAME: lassen
    SPACK_REPO: spack_repos/spack_${SYSTEM_NAME}.git

    # This is based on the assumption that each runner will only ever
    # be able to run one pipeline on a given cluster at one time.
    SPACK_ENV_BASE_NAME: gitlab-${CI_COMMIT_BRANCH}-${GITLAB_USER_LOGIN}-${SYSTEM_NAME}-${CI_RUNNER_SHORT_TOKEN}

    # This is needed to ensure that we run as lbannusr.
    LLNL_SERVICE_USER: lbannusr

    # This needs to be imported here, too. Failure to do so causes
    # problems if it's not set.
    LBANN_CI_RUN_WEEKLY: ${LBANN_CI_RUN_WEEKLY}

    # Preserve any spack-build-* directories to allow for faster testing
    GIT_CLEAN_FLAGS: -ffdx -e spack-build-*

  tags:
    - lassen
    - shell
