support for auto enabling gpu tests

pr_testing/test_multiple_prs.sh

@@ -69,6 +69,35 @@ function should_skip_test()
   return 1
 }
 
+function should_enable_gpu_tests()
+{
+  local file="$1"
+  local patterns=(
+    '^([^/]+/){3}alpaka/'
+    '^HeterogeneousCore/Common/'
+    '^HeterogeneousCore/CUDA.*/'
+    '^HeterogeneousCore/ROCm.*/'
+    '^HeterogeneousCore/Alpaka.*/'
+    '^HeterogeneousTest/'
+    '^HLTrigger/Configuration/'
+    '^Configuration/PyReleaseValidation/'
+  )
+  while IFS= read -r filename; do
+    for regexp in "${patterns[@]}"; do
+      if [[ "$filename" =~ $regexp ]]; then
+	    echo "File with GPU code found: $filename"
+        return 0
+      fi
+    done
+    # A BuildFile.xml containing a CUDA/ROCm/Alpaka dependency
+    if [[ "$filename" == */BuildFile.xml && -f "$filename" ]] && grep -Eq '[[:space:]]name[[:space:]]*=[[:space:]]*"(cuda|rocm|alpaka)"' "$filename"; then
+	  echo "Dependency on cuda/rocm/alpaka found: $filename"
+      return 0
+    fi
+  done < "$file"
+  return 1
+}
+
 # Function to extract filenames by headername and append to indirectly-changed-files.txt
 function extract_filenames() {
   local headername="$1"
@@ -113,13 +142,12 @@ CACHED=${WORKSPACE}/CACHED            # Where cached PR metada etc are kept
 PR_TESTING_DIR=${CMS_BOT_DIR}/pr_testing
 COMMON=${CMS_BOT_DIR}/common
 CONFIG_MAP=$CMS_BOT_DIR/config.map
-[ "${USE_IB_TAG}" != "true" ] && export USE_IB_TAG=false
-readarray -t REQUIRED_GPU_TYPES < <(tr -d '\r' < "${CMS_BOT_DIR}/gpu_flavors.txt")
-readarray -t ONDEMAND_GPU_TYPES < <(tr -d '\r' < "${CMS_BOT_DIR}/gpu_flavors_ondemand.txt")
+[ "${USE_IB_TAG}" != "true" ] && export USE_IB_TAG="false"
+readarray -t REQUIRED_GPU_TYPES < <(tr -d '\r' < "${CMS_BOT_DIR}/gpu_flavors.txt" | tr '[:lower:]' '[:upper:]')
+readarray -t ONDEMAND_GPU_TYPES < <(tr -d '\r' < "${CMS_BOT_DIR}/gpu_flavors_ondemand.txt" | tr '[:lower:]' '[:upper:]')
 ALL_GPU_TYPES=( ${REQUIRED_GPU_TYPES[@]} ${ONDEMAND_GPU_TYPES[@]} )
 
-[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="THREADING HIGH_STATS NANO $(echo ${ALL_GPU_TYPES[@]} | tr '[a-z]' '[A-Z]')"
-EXTRA_RELVALS_TESTS=$(echo ${EXTRA_RELVALS_TESTS} | tr ' ' '\n' | grep -v THREADING | grep -v RNTUPLE | grep -v GPU | tr '\n' ' ')
+EXTRA_RELVALS_TESTS=$(echo "HIGH_STATS NANO ${EXTRA_RELVALS_TESTS} ${ALL_GPU_TYPES[*]}" | tr ' ' '\n' | grep -v THREADING | grep -v RNTUPLE | grep -v GPU | sort | grep -v '^$' | uniq | tr '\n' ' ')
 # ---
 # doc: Input variable
 # PULL_REQUESTS   # "cms-sw/cmsdist#4488,cms-sw/cmsdist#4480,cms-sw/cmsdist#4479,cms-sw/root#116"
@@ -194,19 +222,6 @@ if [ $(echo "${CONFIG_LINE}" | grep "PROD_ARCH=1" | wc -l) -gt 0 ] ; then
   fi
 fi
 
-IFS=',' read -ra SELECTED_GPU_TYPES <<< "$SELECTED_GPU_TYPES"
-
-for gpu_type in ${SELECTED_GPU_TYPES[@]} ; do
-  VAR_NAME="MATRIX_EXTRAS_${gpu_type}"
-  if [ -z "${!VAR_NAME}" ]; then
-    eval "$VAR_NAME=\"${MATRIX_EXTRAS_GPU}\""
-  fi
-  VAR_NAME="EXTRA_MATRIX_ARGS_${gpu_type}"
-  if [ -z "${!VAR_NAME}" ]; then
-    eval "$VAR_NAME=\"${EXTRA_MATRIX_ARGS_GPU}\""
-  fi
-done
-
 if [ "${BUILD_ONLY}" = "true" ] ; then
   DO_COMPARISON=false
 fi
@@ -235,7 +250,7 @@ fi
 
 DISABLE_CMS_DEPRECATED=false
 DISABLE_GPU_TESTS=true
-if [ $(uname -m) != "aarch64" ] ; then
+if [ $(uname -m) == "x86_64" ] ; then
   DISABLE_GPU_TESTS=false
 fi
 CMSSW_QUEUE=$(echo ${RELEASE_FORMAT} | sed 's/_X.*/_X/')  # RELEASE_FORMAT - CMSSW_10_4_X_2018-11-26-2300
@@ -289,6 +304,7 @@ if [ $(echo ${UNIQ_REPO_NAMES_WITH_COUNT}  | grep -v '1 ' | wc -w ) -gt 0 ]; the
 fi
 
 # Filter PR for specific repo and then check if its PRs point to same base branch
+if [ $(echo "${UNIQ_REPOS}" | tr ' ' '\n' | sed 's|.*/||' | grep -E '^(cmsdist|pkgtools)$' | wc -l) -gt 0 ] ; then USE_IB_TAG="false"; fi
 for U_REPO in ${UNIQ_REPOS}; do
     FILTERED_PRS=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep ${U_REPO} | tr '\n' ' ' )
     MASTER_LIST=""
@@ -361,6 +377,39 @@ fi
 
 WORKFLOWS_PR_LABELS=""
 scram -a $SCRAM_ARCH project $CMSSW_IB
+pushd $CMSSW_IB
+  set +x
+  eval $(scram runtime -sh)
+  if $USE_IB_TAG ; then
+    git cms-init --upstream-only $CMSSW_IB
+  else
+    git cms-init --upstream-only
+  fi
+  mv src src.init
+  if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^GPU$' | wc -l) -eq 0 ] ; then
+    rsync -a src.init/ src/
+    MERGE_OK=false
+    for PR in $(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "cms-sw/cmssw#"); do
+      MERGE_OK=true
+      PR_NR=$(echo ${PR} | sed 's/.*#//')
+      if ! git cms-merge-topic --ssh -u ${PR_NR} ; then
+        MERGE_OK=false
+        break
+      fi
+    done
+    if $MERGE_OK ; then
+      (cd src && git diff --name-only $CMSSW_VERSION > $WORKSPACE/cmssw-changed-files.txt)
+      if should_enable_gpu_tests $WORKSPACE/cmssw-changed-files.txt ; then
+        SELECTED_GPU_TYPES=$(IFS=,; echo "${ALL_GPU_TYPES[*]}")
+        ENABLE_BOT_TESTS=$(echo "${ENABLE_BOT_TESTS} ${ALL_GPU_TYPES[*]}" | tr ' ' '\n' | sort | grep -v '^$' | uniq | tr '\n' ' ')
+        echo "Auto enabled GPU tests: ${SELECTED_GPU_TYPES}"
+      fi
+    fi
+    rm -rf src; mkdir src
+  fi
+  eval $(scram unset -sh)
+  set -x
+popd
 
 if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^HLT_P2_TIMING$' | wc -l) -gt 0 ] ; then
   if [ "$(uname -m)" = "x86_64" ] ; then
@@ -369,6 +418,19 @@ if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^HLT_P2_TIMIN
     fi
   fi
 fi
+
+IFS=',' read -ra SELECTED_GPU_TYPES <<< "$SELECTED_GPU_TYPES"
+for gpu_type in ${SELECTED_GPU_TYPES[@]} ; do
+  VAR_NAME="MATRIX_EXTRAS_${gpu_type}"
+  if [ -z "${!VAR_NAME}" ]; then
+    eval "$VAR_NAME=\"${MATRIX_EXTRAS_GPU}\""
+  fi
+  VAR_NAME="EXTRA_MATRIX_ARGS_${gpu_type}"
+  if [ -z "${!VAR_NAME}" ]; then
+    eval "$VAR_NAME=\"${EXTRA_MATRIX_ARGS_GPU}\""
+  fi
+done
+
 if $DO_COMPARISON ; then
   mkdir $WORKSPACE/ib-baseline-tests
   pushd $WORKSPACE/ib-baseline-tests
@@ -548,7 +610,6 @@ TEST_DASGOCLIENT=false
 SKIP_STATIC_CHECKS=false
 [ $(echo ",${SKIP_TESTS}," | grep ',static,' | wc -l) -gt 0 ] && SKIP_STATIC_CHECKS=true
 if ${BUILD_EXTERNAL} ; then
-    export USE_IB_TAG=false
     mark_commit_status_all_prs '' 'pending' -u "${BUILD_URL}" -d "Building CMSSW externals" || true
     if [ ! -d "pkgtools" ] ; then
         git clone git@github.com:cms-sw/pkgtools -b $PKG_TOOL_BRANCH
@@ -771,7 +832,7 @@ if ${BUILD_EXTERNAL} ; then
       fi
     fi
     rm -rf scram-buildrules
-    cd $WORKSPACE/$CMSSW_IB/src
+    cd $WORKSPACE/$CMSSW_IB
     touch $WORKSPACE/cmsswtoolconf.log
     CTOOLS=$WORKSPACE/$CMSSW_IB/config/toolbox/${ARCHITECTURE}/tools/selected
     BTOOLS=${CTOOLS}.backup
@@ -863,7 +924,7 @@ if ${BUILD_EXTERNAL} ; then
           echo "  Tool changed/updated: ${name}"
         fi
       done
-      sed -i -e 's|.*/lib/python2.7/site-packages" .*||;s|.*/lib/python3.6/site-packages" .*||' ../config/Self.xml
+      sed -i -e 's|.*/lib/python2.7/site-packages" .*||;s|.*/lib/python3.6/site-packages" .*||' config/Self.xml
       touch $CTOOLS/*.xml
       (scram setup && scram setup self && rm -rf $WORKSPACE/$CMSSW_IB/external && scram build -r echo_CXX) >> $WORKSPACE/scram-tool-setup.log 2>&1 || TOOL_SETUP=false
       echo "DEP_NAMES=${DEP_NAMES}"
@@ -900,7 +961,7 @@ if ${BUILD_EXTERNAL} ; then
     if [ -e $WORKSPACE/$CMSSW_IB/config/SCRAM/hooks/runtime/00-nvidia-drivers ] ; then
       SCRAM=scram bash -ex $WORKSPACE/$CMSSW_IB/config/SCRAM/hooks/runtime/00-nvidia-drivers || true
     fi
-    git cms-init --upstream-only
+    rsync -a src.init/ src/
     pushd $WORKSPACE/$CMSSW_IB/src
       if [ "X$BUILD_FULL_CMSSW" = "Xtrue" ] ; then
         git checkout $(echo "${CONFIG_LINE}" | sed 's|.*RELEASE_BRANCH=||;s|;.*||')
@@ -933,6 +994,7 @@ PYTHON3_BUILD_OK=true
 RUN_TESTS=true
 
 cd $WORKSPACE/$CMSSW_IB
+[ -d src/.git ] || rsync -a src.init/ src/
 
 set +x
 eval $(scram run -sh)
@@ -958,8 +1020,6 @@ echo '{}' > $RECENT_COMMITS_FILE
 touch $WORKSPACE/changed-files
 if [ ! -d $WORKSPACE/cms-prs ]  ; then git clone --depth 1 git@github.com:cms-sw/cms-prs $WORKSPACE/cms-prs ; fi
 if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
-  if $USE_IB_TAG ; then git cms-init --upstream-only $CMSSW_IB ; fi
-
   # this is to test several pull requests at the same time
   for PR in $( echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#"); do
     echo 'I will add the following pull request to the test'
@@ -998,10 +1058,9 @@ if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
 
   if [[ "${PRODUCTION_RELEASE}" == "true" && "${PULL_REQUEST}" == *"/cmssw#"* ]]; then
     pushd ${CMSSW_BASE}
-      mv src src.tmp && mkdir src
-      cd src
+      mv src src.tmp && rsync -a src.init/ src/ && cd src
       THRDS=""
-      git cms-init --upstream-only && git checkout -b codechecks $CMSSW_IB
+      git checkout -b codechecks $CMSSW_IB
       git repack -h 2>&1 | grep '\-\-threads' && THRDS="--threads ${NCPU}" || true
       git repack -a -d ${THRDS}
       git repack -a -d ${THRDS}
@@ -1034,6 +1093,7 @@ elif [ "X$BUILD_FULL_CMSSW" = "Xtrue" ] ; then
   echo "##### CMSSW Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
   git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
 fi
+rm -rf $WORKSPACE/$CMSSW_IB/src.init
 if ! scram build -r echo_CXX > $WORKSPACE/build.log 2>&1 ; then
     echo "**ERROR**: SCRAM failed to generate build rules, there might be syntax errors in modified BuildFiles." > ${RESULTS_DIR}/10-report.res
     echo "SCRAM_BUILD_CXX;ERROR,SCRAM Build Rules,See Log,build.log" > ${RESULTS_DIR}/scramb.txt