From cf6f2f039d8e1c98bdd9690357e2b4670f23ae4e Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Thu, 9 May 2019 16:45:22 -0500
Subject: [PATCH 1/6] Adding cross-compilation for theta (ANL) and qsub
 (COBALT) batch controls.

---
 MANIFEST                                      |  2 +
 bin/batch/ctest.qsub.in.cmake                 | 21 ++++++++
 config/cmake/scripts/CTestScript.cmake        | 12 ++++-
 config/cmake/scripts/HDF5config.cmake         |  2 +-
 .../cmake/scripts/HPC/qsub-HDF5options.cmake  | 49 +++++++++++++++++++
 5 files changed, 83 insertions(+), 3 deletions(-)
 create mode 100755 bin/batch/ctest.qsub.in.cmake
 create mode 100644 config/cmake/scripts/HPC/qsub-HDF5options.cmake

diff --git a/MANIFEST b/MANIFEST
index 41edffb164..5021503201 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -103,6 +103,7 @@
 ./bin/batch/knl_ctestP.sl.in.cmake
 ./bin/batch/knl_ctestS.sl.in.cmake
 ./bin/batch/knl_H5detect.sl.in.cmake
+./bin/batch/ctest.qsub.in.cmake
 ./bin/batch/ray_ctestP.lsf.in.cmake
 ./bin/batch/ray_ctestS.lsf.in.cmake
 ./bin/batch/raybsub
@@ -3407,6 +3408,7 @@
 # CMake-specific HPC Scripts
 ./config/cmake/scripts/HPC/sbatch-HDF5options.cmake
 ./config/cmake/scripts/HPC/bsub-HDF5options.cmake
+./config/cmake/scripts/HPC/qsub-HDF5options.cmake
 ./config/cmake/scripts/HPC/raybsub-HDF5options.cmake
 
 # Files generated by autogen
diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
new file mode 100755
index 0000000000..f1d9b0091d
--- /dev/null
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -0,0 +1,21 @@
+#!/bin/bash -l
+if [ $# -gt 0 ]; then
+    SUMMARY_FILE=$1
+fi
+ACCOUNT_ID=@ACCOUNT_ID@
+
+echo "Run parallel test command. Test output will be in build/${SUMMARY_FILE}"
+CTEST_CMD=`which ctest`
+
+#SKIPTESTS <<KEYWORD:script inserts list of skips tests here -- don't remove>>
+
+cd @HDF5_BINARY_DIR@
+if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
+  CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"      
+else
+  CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test " 
+fi
+
+qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+
+echo "Done running ctest parallel command."
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index e234057280..e1fe85a04f 100644
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -280,7 +280,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
         if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
           execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
         else ()
-          execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+          if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+            execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestS.out)
+          else ()
+            execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+          endif()
         endif ()
         message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
         execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
@@ -299,7 +303,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
           if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
             execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
           else ()
-            execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+            if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+              execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestP.out)
+            else ()
+              execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+            endif ()
           endif ()
           message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
           execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake
index 0d85b4225d..3ae48ce883 100644
--- a/config/cmake/scripts/HDF5config.cmake
+++ b/config/cmake/scripts/HDF5config.cmake
@@ -44,7 +44,7 @@ set (CTEST_SOURCE_VERSEXT "")
 #CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo
 #CTEST_SOURCE_NAME - name of source folder; HDF5-1.10.0
 #MODEL - CDash group name
-#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub
+#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub, qsub
 #MPI - enable MPI;
 if (DEFINED CTEST_SCRIPT_ARG)
     # transform ctest script arguments of the form
diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
new file mode 100644
index 0000000000..d131e4f0d0
--- /dev/null
+++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
@@ -0,0 +1,49 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+####  Change default configuration of options in config/cmake/cacheinit.cmake file        ###
+####  format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ")                 ###
+#############################################################################################
+
+### uncomment/comment and change the following lines for other configuration options
+
+#############################################################################################
+### enable parallel builds
+if (DEFINED MPI)
+  # maximum parallel processor count for build and test       ####
+  set (MAX_PROC_COUNT 8)
+  set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=ON")
+  set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
+  set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
+  set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=OFF")
+endif ()
+#############################################################################################
+### options to run test scripts in batch commands
+set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub")
+if (DEFINED KNL)
+  ### some additions and alternatives to cross compile on haswell for knl
+  set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+  set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+endif ()
+set (LOCAL_BATCH_SCRIPT_COMMAND "qsub")
+set (LOCAL_BATCH_TEST "TRUE")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun")
+# Option to suppress writing job statistics; to avoid issues with h5diff comparisons.
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DACCOUNT_ID:STRING=${LOCAL_BATCH_SCRIPT_ARGS}")
+
+#############################################################################################
+#############################################################################################

From aaacc50f64a4ff4c02764b706ac9884f6ff42754 Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Fri, 10 May 2019 12:13:25 -0500
Subject: [PATCH 2/6] fixed missing paralle test

---
 config/cmake/scripts/HPC/qsub-HDF5options.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
index d131e4f0d0..51c3891a58 100644
--- a/config/cmake/scripts/HPC/qsub-HDF5options.cmake
+++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
@@ -29,6 +29,7 @@ endif ()
 #############################################################################################
 ### options to run test scripts in batch commands
 set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub")
+set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctest.qsub")
 if (DEFINED KNL)
   ### some additions and alternatives to cross compile on haswell for knl
   set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
@@ -38,6 +39,7 @@ set (LOCAL_BATCH_SCRIPT_COMMAND "qsub")
 set (LOCAL_BATCH_TEST "TRUE")
 set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
 set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}")
 set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun")
 # Option to suppress writing job statistics; to avoid issues with h5diff comparisons.
 set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q")

From 19d2cea9aec648eb3c609dee4d2dc9baa101c967 Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Fri, 10 May 2019 14:34:56 -0500
Subject: [PATCH 3/6] added -M HPC to ctest command

---
 MANIFEST                              |  4 ----
 bin/batch/cori_ctestP.sl.in.cmake     | 17 -----------------
 bin/batch/cori_ctestS.sl.in.cmake     | 16 ----------------
 bin/batch/cori_knl_ctestP.sl.in.cmake | 17 -----------------
 bin/batch/cori_knl_ctestS.sl.in.cmake | 16 ----------------
 bin/batch/ctest.qsub.in.cmake         |  4 ++--
 bin/batch/ctestP.lsf.in.cmake         |  2 +-
 bin/batch/ctestP.sl.in.cmake          |  2 +-
 bin/batch/ctestS.lsf.in.cmake         |  2 +-
 bin/batch/ctestS.sl.in.cmake          |  2 +-
 bin/batch/knl_ctestP.sl.in.cmake      |  2 +-
 bin/batch/knl_ctestS.sl.in.cmake      |  2 +-
 bin/batch/ray_ctestP.lsf.in.cmake     |  2 +-
 bin/batch/ray_ctestS.lsf.in.cmake     |  2 +-
 14 files changed, 10 insertions(+), 80 deletions(-)
 delete mode 100644 bin/batch/cori_ctestP.sl.in.cmake
 delete mode 100644 bin/batch/cori_ctestS.sl.in.cmake
 delete mode 100644 bin/batch/cori_knl_ctestP.sl.in.cmake
 delete mode 100644 bin/batch/cori_knl_ctestS.sl.in.cmake

diff --git a/MANIFEST b/MANIFEST
index 5021503201..c3dc0e9cf1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -92,10 +92,6 @@
 ./bin/timekeeper      _DO_NOT_DISTRIBUTE_
 ./bin/trace
 ./bin/yodconfigure
-./bin/batch/cori_ctestP.sl.in.cmake
-./bin/batch/cori_ctestS.sl.in.cmake
-./bin/batch/cori_knl_ctestP.sl.in.cmake
-./bin/batch/cori_knl_ctestS.sl.in.cmake
 ./bin/batch/ctestP.lsf.in.cmake
 ./bin/batch/ctestP.sl.in.cmake
 ./bin/batch/ctestS.lsf.in.cmake
diff --git a/bin/batch/cori_ctestP.sl.in.cmake b/bin/batch/cori_ctestP.sl.in.cmake
deleted file mode 100644
index 287fe3c1a2..0000000000
--- a/bin/batch/cori_ctestP.sl.in.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-#SBATCH -C haswell
-#SBATCH --nodes=1
-#SBATCH -t 00:30:00
-#SBATCH --mail-type=BEGIN,END,FAIL
-##SBATCH --mail-user=<username>@sandia.gov
-#SBATCH --export=ALL
-#SBATCH --job-name=h5_ctestP
-
-cd @HDF5_BINARY_DIR@
-CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
-
-echo "Run $CMD. Test output will be in build/ctestP.out"
-$CMD >& ctestP.out
-echo "Done running $CMD"
-
diff --git a/bin/batch/cori_ctestS.sl.in.cmake b/bin/batch/cori_ctestS.sl.in.cmake
deleted file mode 100644
index 50bef6b1ab..0000000000
--- a/bin/batch/cori_ctestS.sl.in.cmake
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-#SBATCH -C haswell
-#SBATCH --nodes=1
-#SBATCH -t 00:30:00
-#SBATCH --mail-type=BEGIN,END,FAIL
-##SBATCH --mail-user=<username>@sandia.gov
-#SBATCH --export=ALL
-#SBATCH --job-name=h5_ctestS
-
-cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
-
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
diff --git a/bin/batch/cori_knl_ctestP.sl.in.cmake b/bin/batch/cori_knl_ctestP.sl.in.cmake
deleted file mode 100644
index 82378ec4e9..0000000000
--- a/bin/batch/cori_knl_ctestP.sl.in.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-#SBATCH -C knl,quad,cache
-#SBATCH --nodes=1
-#SBATCH -t 00:30:00
-#SBATCH --mail-type=BEGIN,END,FAIL
-##SBATCH --mail-user=<username>@sandia.gov
-#SBATCH --export=ALL
-#SBATCH --job-name=h5_ctestP
-
-cd @HDF5_BINARY_DIR@
-CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
-
-echo "Run $CMD. Test output will be in build/ctestP.out"
-$CMD >& ctestP.out
-echo "Done running $CMD"
-
diff --git a/bin/batch/cori_knl_ctestS.sl.in.cmake b/bin/batch/cori_knl_ctestS.sl.in.cmake
deleted file mode 100644
index c6d9e833e2..0000000000
--- a/bin/batch/cori_knl_ctestS.sl.in.cmake
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-#SBATCH -C knl,quad,cache
-#SBATCH --nodes=1
-#SBATCH -t 00:30:00
-#SBATCH --mail-type=BEGIN,END,FAIL
-##SBATCH --mail-user=<username>@sandia.gov
-#SBATCH --export=ALL
-#SBATCH --job-name=h5_ctestS
-
-cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
-
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
index f1d9b0091d..8ea774096d 100755
--- a/bin/batch/ctest.qsub.in.cmake
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -11,9 +11,9 @@ CTEST_CMD=`which ctest`
 
 cd @HDF5_BINARY_DIR@
 if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
-  CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"      
+  CMD="${CTEST_CMD} . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
 else
-  CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test " 
+  CMD="${CTEST_CMD} . -M HPC -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test"
 fi
 
 qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake
index 2777f39cad..26d4bc8dfe 100644
--- a/bin/batch/ctestP.lsf.in.cmake
+++ b/bin/batch/ctestP.lsf.in.cmake
@@ -14,6 +14,6 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R 'TEST_PAR|PH5DIFF|PERFORM' -E t_cache_image -C Release -T test >& ctestP.out
+ctest . -M HPC -R 'TEST_PAR|PH5DIFF|PERFORM' -C Release -T test >& ctestP.out
 
 echo "Done running ctest parallel command."
diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake
index 592d37dfe2..9d5ff70e57 100644
--- a/bin/batch/ctestP.sl.in.cmake
+++ b/bin/batch/ctestP.sl.in.cmake
@@ -8,7 +8,7 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test >& ctestP.out
+ctest . -M HPC -R MPI_TEST_ -C Release -T test >& ctestP.out
 
 echo "Done running ctestP.sl" 
 
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index 7ef5af58b7..627bd7d001 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
+ctest . -M HPC -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."
diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake
index 4f96bb940f..eb116facb2 100644
--- a/bin/batch/ctestS.sl.in.cmake
+++ b/bin/batch/ctestS.sl.in.cmake
@@ -8,7 +8,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+CMD="ctest . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake
index 73c5ac597b..53886dc7cc 100644
--- a/bin/batch/knl_ctestP.sl.in.cmake
+++ b/bin/batch/knl_ctestP.sl.in.cmake
@@ -10,7 +10,7 @@
 
 cd @HDF5_BINARY_DIR@
 #run parallel tests except t_cache_image test
-ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test >& ctestP.out
+ctest . -M HPC -R MPI_TEST_ -C Release -T test >& ctestP.out
 
 echo "Done running $CMD"
 
diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake
index af6353bae3..453380b731 100644
--- a/bin/batch/knl_ctestS.sl.in.cmake
+++ b/bin/batch/knl_ctestS.sl.in.cmake
@@ -9,7 +9,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+CMD="ctest . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake
index 082276a1bc..80b4409a77 100644
--- a/bin/batch/ray_ctestP.lsf.in.cmake
+++ b/bin/batch/ray_ctestP.lsf.in.cmake
@@ -15,6 +15,6 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R 'MPI_TEST_' -E t_cache_image -C Release -T test >& ctestP.out
+ctest . -M HPC -R 'MPI_TEST_' -C Release -T test >& ctestP.out
 
 echo "Done running ctest parallel command."
diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake
index da2043824e..773bd687b0 100644
--- a/bin/batch/ray_ctestS.lsf.in.cmake
+++ b/bin/batch/ray_ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
+ctest . -M HPC -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."

From 44fdc3bf96d1dc3a804238916975efde67b55f12 Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Fri, 10 May 2019 15:45:47 -0500
Subject: [PATCH 4/6] remove -M option

---
 bin/batch/ctest.qsub.in.cmake     | 4 ++--
 bin/batch/ctestP.lsf.in.cmake     | 2 +-
 bin/batch/ctestP.sl.in.cmake      | 2 +-
 bin/batch/ctestS.lsf.in.cmake     | 2 +-
 bin/batch/ctestS.sl.in.cmake      | 2 +-
 bin/batch/knl_ctestP.sl.in.cmake  | 2 +-
 bin/batch/knl_ctestS.sl.in.cmake  | 2 +-
 bin/batch/ray_ctestP.lsf.in.cmake | 2 +-
 bin/batch/ray_ctestS.lsf.in.cmake | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
index 8ea774096d..702fca7f01 100755
--- a/bin/batch/ctest.qsub.in.cmake
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -11,9 +11,9 @@ CTEST_CMD=`which ctest`
 
 cd @HDF5_BINARY_DIR@
 if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
-  CMD="${CTEST_CMD} . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
+  CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"
 else
-  CMD="${CTEST_CMD} . -M HPC -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test"
+  CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test"
 fi
 
 qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake
index 26d4bc8dfe..d8d9ee012e 100644
--- a/bin/batch/ctestP.lsf.in.cmake
+++ b/bin/batch/ctestP.lsf.in.cmake
@@ -14,6 +14,6 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -M HPC -R 'TEST_PAR|PH5DIFF|PERFORM' -C Release -T test >& ctestP.out
+ctest . -R 'TEST_PAR|PH5DIFF|PERFORM' -C Release -T test >& ctestP.out
 
 echo "Done running ctest parallel command."
diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake
index 9d5ff70e57..6399de7934 100644
--- a/bin/batch/ctestP.sl.in.cmake
+++ b/bin/batch/ctestP.sl.in.cmake
@@ -8,7 +8,7 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-ctest . -M HPC -R MPI_TEST_ -C Release -T test >& ctestP.out
+ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
 
 echo "Done running ctestP.sl" 
 
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index 627bd7d001..7ef5af58b7 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -M HPC -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
+ctest . -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."
diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake
index eb116facb2..4f96bb940f 100644
--- a/bin/batch/ctestS.sl.in.cmake
+++ b/bin/batch/ctestS.sl.in.cmake
@@ -8,7 +8,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake
index 53886dc7cc..f985fbb118 100644
--- a/bin/batch/knl_ctestP.sl.in.cmake
+++ b/bin/batch/knl_ctestP.sl.in.cmake
@@ -10,7 +10,7 @@
 
 cd @HDF5_BINARY_DIR@
 #run parallel tests except t_cache_image test
-ctest . -M HPC -R MPI_TEST_ -C Release -T test >& ctestP.out
+ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
 
 echo "Done running $CMD"
 
diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake
index 453380b731..af6353bae3 100644
--- a/bin/batch/knl_ctestS.sl.in.cmake
+++ b/bin/batch/knl_ctestS.sl.in.cmake
@@ -9,7 +9,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -M HPC -E MPI_TEST_ -C Release -j 32 -T test"
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake
index 80b4409a77..7067a65811 100644
--- a/bin/batch/ray_ctestP.lsf.in.cmake
+++ b/bin/batch/ray_ctestP.lsf.in.cmake
@@ -15,6 +15,6 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -M HPC -R 'MPI_TEST_' -C Release -T test >& ctestP.out
+ctest . -R 'MPI_TEST_' -C Release -T test >& ctestP.out
 
 echo "Done running ctest parallel command."
diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake
index 773bd687b0..da2043824e 100644
--- a/bin/batch/ray_ctestS.lsf.in.cmake
+++ b/bin/batch/ray_ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -M HPC -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
+ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."

From 6546c764d3d39028d2ec61da236cc69b44414127 Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Fri, 10 May 2019 16:07:12 -0500
Subject: [PATCH 5/6] updated -E option

---
 bin/batch/ctestP.lsf.in.cmake | 2 +-
 bin/batch/ctestS.lsf.in.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake
index d8d9ee012e..3fdd5ce99d 100644
--- a/bin/batch/ctestP.lsf.in.cmake
+++ b/bin/batch/ctestP.lsf.in.cmake
@@ -14,6 +14,6 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R 'TEST_PAR|PH5DIFF|PERFORM' -C Release -T test >& ctestP.out
+ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
 
 echo "Done running ctest parallel command."
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index 7ef5af58b7..a01d39b00a 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
+ctest . -E MPI_TEST_ -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."

From 9406278103a7b72d3eacea9fbeaadbf34bf7e43a Mon Sep 17 00:00:00 2001
From: Larry Knox <lrknox@hdfgroup.org>
Date: Tue, 14 May 2019 13:52:25 -0500
Subject: [PATCH 6/6] Reorder lines in HDF5config.cmake so that additions to
 end of $HPC-HDF5options scripts will take precedence.

---
 config/cmake/scripts/HDF5config.cmake | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake
index 3ae48ce883..44ed531275 100644
--- a/config/cmake/scripts/HDF5config.cmake
+++ b/config/cmake/scripts/HDF5config.cmake
@@ -179,7 +179,6 @@ if (NOT DEFINED HPC)
 else ()
   set (CTEST_SITE "${SITE_OS_NAME}")
   set (CTEST_CMAKE_GENERATOR "Unix Makefiles")
-  include (${CTEST_SOURCE_DIRECTORY}/config/cmake/scripts/HPC/${HPC}-HDF5options.cmake)
 endif ()
 ###################################################################
 
@@ -229,6 +228,9 @@ if (WIN32)
 else ()
   set (BINFILEBASE "HDF5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}")
   include (${CTEST_SCRIPT_DIRECTORY}/HDF5options.cmake)
+  if (DEFINED HPC)
+    include (${CTEST_SOURCE_DIRECTORY}/config/cmake/scripts/HPC/${HPC}-HDF5options.cmake)
+  endif ()
   include (${CTEST_SCRIPT_DIRECTORY}/CTestScript.cmake)
   if (APPLE)
     if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.dmg")