.gitlab-ci.yml

# this include allows us to reference defaults in anl/ci-resource/defaults

include:
  - project: 'anl/ci-resources/defaults'
    ref: main
    file:
      - '/runners.yml'

workflow:
    rules:
      - if: $CI_PIPELINE_SOURCE == "trigger"   # triggered from GitHub via /run-aurora-ci

variables:
  PROJ: "chipStar_test"
  QUEUE: "debug"
  TIME: "1:00:00"
  WRK_DIR: "/lus/flare/projects/${PROJ}/chipStar/pipeline_${CI_PIPELINE_ID}"
  ANL_AURORA_SCHEDULER_PARAMETERS: "-q ${QUEUE} -l select=1,walltime=${TIME},filesystems=home -A ${PROJ}"
  GENESIS_CI_PROJECT_DIR: "/home/bertoni/projects/p01.chipStar/GENESIS-Share-CI/"

stages:
    - .pre
    - build_chipstar
    - builds_chipstar_install_script_llvm22
    - runs_tests_chipstar_install_script_llvm22
    - builds_chipstar_aurora_script_llvm21
    - runs_chipstar_aurora_script_llvm21
    - builds_chipstar_aurora_script_llvm22
    - runs_chipstar_aurora_script_llvm22
    - builds_hoMusic
    - runs_hoMusic
    - performance_hoMusic
    - builds_zeroRK
    - runs_zeroRK
    - correctness_zeroRK_hydrogen
    - correctness_zeroRK_box_reactor
    - performance_zeroRK
    - builds_opensn
    - runs_opensn
    - performance_opensn
    - cleanup

setup:
  stage: .pre
  extends: .aurora-shell-runner
  resource_group: aurora_pipeline  
  script:
    # Remove pipeline directories older than 7 days
    - find /lus/flare/projects/${PROJ}/chipStar -maxdepth 1 -name "pipeline_*" -mtime +7 -exec rm -rf {} \;
    - mkdir -p ${WRK_DIR}
    - cd ${WRK_DIR}
      # setup chipStar
    - git clone https://github.com/CHIP-SPV/chipStar.git
    - cd chipStar
    - |
      if [ -n "${GITHUB_PR_NUMBER}" ]; then
         git fetch origin pull/${GITHUB_PR_NUMBER}/head
         git checkout FETCH_HEAD
      else
         git checkout ${GITHUB_SHA}
      fi
    - git submodule update --init --recursive
    - module list

# testing with hand script
aurora_build_test:
  stage: build_chipstar
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - cd ${WRK_DIR}/chipStar
    # first test with already installed clang. will update to patch and build
    - module use /soft/modulefiles
    - module load cmake chipStar/.llvm21/20260808-21/release
    - rm -rf build
    - mkdir build
    - cd build
    - cmake -DCMAKE_BUILD_TYPE=Release -DCHIP_BUILD_SAMPLES=ON -DCMAKE_INSTALL_PREFIX=${WRK_DIR}/install_aurora ..
    - gitcommit=$(git rev-parse HEAD)
    - make -j48
    - make install
    - cd ..
    - rm -rf build

chipStar_script_install_llvm22:
  stage: builds_chipstar_install_script_llvm22
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - cp -r ${WRK_DIR}/chipStar  ${WRK_DIR}/chipStar_llvm22
    - cd ${WRK_DIR}/chipStar_llvm22
    - module use /soft/modulefiles
    - module load cmake chipStar/.llvm22/20260311-22/release
    - sed -i '/name="chipstar",/a\        cmake_flags=["-DCHIP_LZ_API_QUERY_QUEUE_EMPTY=ON","-DHIPCC_VERIFY_DEVICES=pvc"],' ./install_chipstar.py
    - ./install_chipstar.py --all --install-dir ${WRK_DIR}/install/ --staging-dir $PWD/stage

chipStar_script_test_llvm22:
  stage: runs_tests_chipstar_install_script_llvm22
  resource_group: aurora_pipeline  
  extends: .aurora-batch-runner
  script:
    - cd ${WRK_DIR}/chipStar_llvm22/build
    - module use /soft/modulefiles
    - module load cmake chipStar/.llvm22/20260311-22/release
    - rm -rf $HOME/.cache/chipStar
    - CHIP_BE=level0 make build_tests -j
    - CHIP_MODULE_CACHE_DIR="" ZE_AFFINITY_MASK=0.0 ../scripts/check.py ./ dgpu level0 --timeout 60 --num-tries=1 --num-threads=1

# testing with install script
chipStar_aurora_script_test:
  stage: builds_chipstar_aurora_script_llvm21
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - mkdir ${WRK_DIR}/script_llvm21
    - cp -r ${WRK_DIR}/chipStar ${WRK_DIR}/script_llvm21
    - NO_FRESH_CHIPSTAR_PULL=true temp=${WRK_DIR}/script_llvm21 MODULE_INSTALL_DIR=${WRK_DIR}/modulefiles CHIPSTAR_INSTALL_DIR=${WRK_DIR}/latest_llvm21/ /lus/flare/projects/Aurora_deployment/bertoni/projects/p02.chipStar/chip-star-patch.sh release chipStar/.llvm21/20260808-21/release
    - cd ${WRK_DIR}/modulefiles/chipStar
    - echo 'module_version("llvm21", "default")' > .modulerc.lua
  
chipStar_aurora_script_test_run:
  stage: runs_chipstar_aurora_script_llvm21
  resource_group: aurora_pipeline  
  extends: .aurora-batch-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - cd ${WRK_DIR}/script_llvm21/chipStar/build
    - rm -rf $HOME/.cache/chipStar
    - CHIP_BE=level0 make build_tests -j
    - module load cmake
    - CHIP_MODULE_CACHE_DIR="" ZE_AFFINITY_MASK=0.0 ../scripts/check.py ./ dgpu level0 --timeout 60 --num-tries=1 --num-threads=1
  
chipStar_aurora_script_test_llvm22:
  stage: builds_chipstar_aurora_script_llvm22
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - mkdir ${WRK_DIR}/script_llvm_22
    - cp -r ${WRK_DIR}/chipStar ${WRK_DIR}/script_llvm_22
    - NO_FRESH_CHIPSTAR_PULL=true llvm_version=22 temp=${WRK_DIR}/script_llvm_22/ MODULE_INSTALL_DIR=${WRK_DIR}/modulefiles CHIPSTAR_INSTALL_DIR=${WRK_DIR}/latest_llvm22/ LLVM_INSTALL_DIR=${WRK_DIR}/llvm22 /lus/flare/projects/Aurora_deployment/bertoni/projects/p02.chipStar/chip-star-patch.sh release
  
chipStar_aurora_script_test_llvm22_run:
  stage: runs_chipstar_aurora_script_llvm22
  resource_group: aurora_pipeline  
  extends: .aurora-batch-runner
  script:
    - hostname
    - echo "Running on $(hostname) with setuid shell runner"
    - cd ${WRK_DIR}/script_llvm_22/chipStar/build
    - module load cmake
    - rm -rf $HOME/.cache/chipStar
    - CHIP_BE=level0 make build_tests -j
    - CHIP_MODULE_CACHE_DIR="" ZE_AFFINITY_MASK=0.0 ../scripts/check.py ./ dgpu level0 --timeout 60 --num-tries=1 --num-threads=1

# hoMusic application build and run
hoMusic_build:
  stage: builds_hoMusic
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
    - ${GENESIS_CI_PROJECT_DIR}/hoMusic_ci.sh build chipStar/llvm21
  
hoMusic_run:
  stage: runs_hoMusic
  resource_group: aurora_pipeline  
  extends: .aurora-batch-runner
  script:
    - set -o pipefail
    - rm -f ${GENESIS_CI_PROJECT_DIR}/output
    - ${GENESIS_CI_PROJECT_DIR}/hoMusic_ci.sh run chipStar/llvm21 |& tee -a ${GENESIS_CI_PROJECT_DIR}/output

performance_hoMusic:
  stage: performance_hoMusic
  extends: .aurora-shell-runner
  script:
    - cd ${GENESIS_CI_PROJECT_DIR}
    - |
      TIME_STR=$(grep "real" output | tail -n 1 | awk '{print $2}')
      echo "Extracted time: $TIME_STR"

      # Convert to total seconds using awk (handles 1m50.315s -> 110.315)
      ACTUAL_TIME=$(echo $TIME_STR | awk -F'[ms]' '{print ($1 * 60) + $2}')
      
      # Define baseline and 6% upper threshold
      BASELINE=90.0
      THRESHOLD=$(echo "$BASELINE * 0.06" | bc -l)
      UPPER_BOUND=$(echo "$BASELINE + $THRESHOLD" | bc -l)

      echo "Actual: $ACTUAL_TIME s | Target: < $BASELINE s + $THRESHOLD s)"

      # Compare using bc (returns 1 for true, 0 for false)
      IS_WITHIN_LIMITS=$(echo "$ACTUAL_TIME <= $UPPER_BOUND" | bc -l)

      if [ "$IS_WITHIN_LIMITS" -eq 1 ]; then
        echo "Performance check passed: $ACTUAL_TIME is within 5% of $BASELINE"
        exit 0
      else
        echo "Performance check failed: $ACTUAL_TIME is above $UPPER_BOUND"
        exit 1
      fi
  
# ZeroRK application build and run
zeroRK_build:
  stage: builds_zeroRK
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
   - cd ${WRK_DIR}
   - git clone git@github.com:CHIP-SPV/zero-rk.git -b hip-chipStar
   - cd zero-rk
   - module use /soft/modulefiles
   - module use ${WRK_DIR}/modulefiles
   - module load chipStar/llvm21 cmake
   - mkdir -p build_aurora
   - cd build_aurora
   - sh ../scripts/build_aurora.sh  --no-module-loads
  
zeroRK_run:
  stage: runs_zeroRK
  resource_group: aurora_pipeline  
  extends: .aurora-batch-runner
  script:
  - cd ${WRK_DIR}/zero-rk
  - cd running_test_aurora
  - mkdir -p logs
  - chmod a+x ./submit_job.sh
  - module use /soft/modulefiles
  - module use ${WRK_DIR}/modulefiles
  - module load chipStar/llvm21
  - rm -f output
  - set -o pipefail
  - ./submit_job.sh |& tee -a output

zeroRK_correctness_hydrogen:
  stage: correctness_zeroRK_hydrogen
  resource_group: aurora_pipeline
  extends: .aurora-batch-runner
  script:
  - cd ${WRK_DIR}/zero-rk
  - module use /soft/modulefiles
  - module use ${WRK_DIR}/modulefiles
  - module load chipStar/llvm21
  # run the CPU-vs-GPU comparison in both LU modes; report both before failing
  - rc=0
  - bash ${WRK_DIR}/chipStar/tests/apps/ZeroRK/zerork_correctness_hydrogen.sh 1 || rc=1
  - bash ${WRK_DIR}/chipStar/tests/apps/ZeroRK/zerork_correctness_hydrogen.sh 0 || rc=1
  - exit $rc

zeroRK_correctness_box_reactor:
  stage: correctness_zeroRK_box_reactor
  resource_group: aurora_pipeline
  extends: .aurora-batch-runner
  script:
  - cd ${WRK_DIR}/zero-rk
  - module use /soft/modulefiles
  - module use ${WRK_DIR}/modulefiles
  - module load chipStar/llvm21
  # run the temperature-trend check in both LU modes; report both before failing
  - rc=0
  - bash ${WRK_DIR}/chipStar/tests/apps/ZeroRK/zerork_correctness_box_reactor.sh 1 || rc=1
  - bash ${WRK_DIR}/chipStar/tests/apps/ZeroRK/zerork_correctness_box_reactor.sh 0 || rc=1
  - exit $rc

zeroRK_performance:
  stage: performance_zeroRK
  resource_group: aurora_pipeline  
  extends: .aurora-shell-runner
  script:
  - cd ${WRK_DIR}/zero-rk/running_test_aurora
  - |
    ACTUAL_TIME=$(grep "Job time in seconds" output | awk '{print $5}')
    echo "Extracted time: $ACTUAL_TIME"
    
    # Define baseline and 6% threshold
    BASELINE=85
    THRESHOLD=$(echo "$BASELINE * 0.10" | bc -l)
    UPPER_BOUND=$(echo "$BASELINE + $THRESHOLD" | bc -l)

    echo "Actual: $ACTUAL_TIME s | Target: < $BASELINE s + $THRESHOLD s)"

    # Compare using bc (returns 1 for true, 0 for false)
    IS_WITHIN_LIMITS=$(echo "$ACTUAL_TIME <= $UPPER_BOUND" | bc -l)

    if [ "$IS_WITHIN_LIMITS" -eq 1 ]; then
      echo "Performance check passed: $ACTUAL_TIME is within 5% of $BASELINE"
      exit 0
    else
      echo "Performance check failed: $ACTUAL_TIME is above $UPPER_BOUND"
      exit 1
    fi

# OpenSn application build and run
opensn_build:
  stage: builds_opensn
  resource_group: aurora_pipeline
  extends: .aurora-shell-runner
  script:
   - cd ${WRK_DIR}
   - module load frameworks/2025.3.1
   - export ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE
   - export ONEAPI_DEVICE_SELECTOR=level_zero:gpu
   - module use /soft/modulefiles
   - module use ${WRK_DIR}/modulefiles
   - module load chipStar/llvm21 cmake
   - export BASE=$PWD
   - git clone git@github.com:Open-Sn/opensn.git
   - cd opensn
   - git checkout d0644cd9c633c6ae5e7110d3f8721641e5ef2982                                          
   - grep -q -- '--download-f2cblaslapack=yes' tools/dependencies/CMakeLists.txt                                                                                                                                                            
   - sed -i 's|--download-f2cblaslapack=yes|--download-f2cblaslapack=/lus/flare/projects/chipStar_test/chipStar/dependencies/f2cblaslapack-3.8.0.q2.tar.gz|' tools/dependencies/CMakeLists.txt 
   - mkdir -p build_deps
   - cd build_deps
   - cmake -DCMAKE_INSTALL_PREFIX=$BASE/dependencies ../tools/dependencies
   - make -j
   - cd ..
   - module use /home/pvelesko/modulefiles
   - module load cmake/4.3.2.lua
   - source $BASE/dependencies/bin/set_opensn_env.sh
   - mkdir build_debug
   - cd build_debug
   - CC=hipcc CXX=hipcc cmake -DOPENSN_WITH_HIP=ON -DCMAKE_HIP_ARCHITECTURES=spirv -DCMAKE_HIP_PLATFORM=spirv -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
   - make -j

opensn_run:
  stage: runs_opensn
  resource_group: aurora_pipeline
  extends: .aurora-batch-runner
  script:
   - cd ${WRK_DIR}/opensn/build_debug
   - cp ${WRK_DIR}/chipStar/tests/apps/OpenSn/strong_scaling.* .
   - cp ${WRK_DIR}/chipStar/tests/apps/OpenSn/xs_168g.xs .
   - module use ${WRK_DIR}/modulefiles
   - module use /soft/modulefiles
   - module load chipStar/llvm21
   - module load hdf5
   - rm -rf ~/.cache/chipStar/
   - rm -f output
   - set -o pipefail
   # first run populates the cache
   - CHIP_LOGLEVEL=off mpiexec --cpu-bind=list:1-8 --env OMP_NUM_THREADS=8 --env OMP_PLACES=cores -n 1 gpu_tile_compact.sh python/opensn -i strong_scaling.py |& tee -a output
   # second (timed) run uses the cache
   - CHIP_LOGLEVEL=off mpiexec --cpu-bind=list:1-8 --env OMP_NUM_THREADS=8 --env OMP_PLACES=cores -n 1 gpu_tile_compact.sh python/opensn -i strong_scaling.py |& tee -a output

opensn_performance:
  stage: performance_opensn
  resource_group: aurora_pipeline
  extends: .aurora-shell-runner
  script:
  - cd ${WRK_DIR}/opensn/build_debug
  - |
    # use the last (cached) run's timing; format is "Elapsed execution time: HH:MM:SS.s"
    TIME_STR=$(grep "Elapsed execution time" output | tail -n 1 | awk '{print $NF}')
    ACTUAL_TIME=$(echo $TIME_STR | awk -F: '{print ($1 * 3600) + ($2 * 60) + $3}')
    echo "Extracted time: $TIME_STR -> $ACTUAL_TIME s"

    # Define baseline and 6% threshold
    BASELINE=29
    THRESHOLD=$(echo "$BASELINE * 0.1" | bc -l)
    UPPER_BOUND=$(echo "$BASELINE + $THRESHOLD" | bc -l)

    echo "Actual: $ACTUAL_TIME s | Target: < $BASELINE s + $THRESHOLD s)"

    IS_WITHIN_LIMITS=$(echo "$ACTUAL_TIME <= $UPPER_BOUND" | bc -l)

    if [ "$IS_WITHIN_LIMITS" -eq 1 ]; then
      echo "Performance check passed: $ACTUAL_TIME is within 6% of $BASELINE"
      exit 0
    else
      echo "Performance check failed: $ACTUAL_TIME is above $UPPER_BOUND"
      exit 1
    fi

cleanup:
  stage: cleanup
  extends: .aurora-shell-runner
  resource_group: aurora_pipeline
  when: on_success
  script:
    - rm -rf ${WRK_DIR}

.report_github_status:
  stage: cleanup
  extends: .aurora-shell-runner
  script:
    - |
      if [ -z "${GITHUB_SHA}" ]; then
        echo "No GITHUB_SHA set, skipping GitHub status report"
        exit 0
      fi
      curl -X POST \
        --fail \
        -H "Authorization: token ${GITHUB_TOKEN}" \
        -H "Accept: application/vnd.github.v3+json" \
        https://api.github.com/repos/CHIP-SPV/chipStar/statuses/${GITHUB_SHA} \
        -d "{\"state\":\"${GITHUB_STATUS}\",\"context\":\"aurora-ci\",\"target_url\":\"${CI_PIPELINE_URL}\"}"

report_success:
  extends: .report_github_status
  when: on_success
  variables:
    GITHUB_STATUS: "success"

report_failure:
  extends: .report_github_status
  when: on_failure
  variables:
    GITHUB_STATUS: "failure"