hdf5/test/testvfdswmr.sh.in

#!/usr/bin/env bash
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the vfd swmr feature.
#
###############################################################################
# VFD SWMR concurrent tests which are modified from existing swmr concurrent tests.
# This is copied and modified from testswmr.sh.in
#
###############################################################################

srcdir=@srcdir@

. ${srcdir}/supervise.subr

###############################################################################
## test parameters
###############################################################################

Nreaders=5              # number of readers to launch
Nrdrs_spa=3             # number of sparse readers to launch
Nrecords=400000         # number of records to write
Nrecs_rem=40000         # number of times to shrink
Nrecs_spa=20000         # number of records to write in the sparse test
Nsecs_add=5             # number of seconds per read interval
Nsecs_rem=3             # number of seconds per read interval
Nsecs_addrem=8          # number of seconds per read interval
nerrors=0
nsofterrors=0		# soft errors are expected to occur some of the time
			# on a couple of nondeterministic tests.

###############################################################################
## test parameters for vfd_swmr_bigset_writer.c and vfd_swmr_group_writer.c
## based on HDF5TestExpress:
## 0:  Exhaustive run: Tests take a long time to run.
## 1:  Default run.
## 2+: Quick run
###############################################################################
if [[ -z $HDF5TestExpress ]]; then    # Set to default when not set
    HDF5TestExpress=1
fi
##
##Default setting
BIGSET_n=25                         # -n option: # of iterations
BIGSET_few_s=20                     # -s option: # of datasets (for few_big test)
BIGSET_many_s=500                   # -s option: # of datasets (for many_small test)
GROUP_n=40                          # -n option: # of groups (for group test)
GROUP_attr_n=1                      # -n option: # of groups (for group attribute test)
GROUP_op_n=1                        # -n option: # of groups (for group attribute test)

if [[ "$HDF5TestExpress" -eq 0 ]] ; then         # Setting for exhaustive run
    BIGSET_n=50
    BIGSET_few_s=40
    BIGSET_many_s=1000
    GROUP_n=400
    GROUP_attr_n=2
    GROUP_op_n=2                       
elif [[ "$HDF5TestExpress" -gt 1 ]]; then        # Setting for quick run
    BIGSET_n=10
    BIGSET_few_s=10
    BIGSET_many_s=100
    GROUP_n=20
fi

###############################################################################
## definitions for message file to coordinate test runs
###############################################################################
WRITER_MESSAGE=VFD_SWMR_WRITER_MESSAGE  # The message file created by writer that the open is complete
                                        # This should be the same as the define in "./swmr_common.h"
MESSAGE_TIMEOUT=300                     # Message timeout length in secs
                                        # This should be the same as the define in "./h5test.h"
 
###############################################################################
## For attrdset test: definitions for fifo files to coordinate test runs
###############################################################################
ATTRDSET_FIFO_WRITER_TO_READER=fifo_attrdset_writer_to_reader
ATTRDSET_FIFO_READER_TO_WRITER=fifo_attrdset_reader_to_writer

###############################################################################
## For dsetops test: definitions for fifo files to coordinate test runs
###############################################################################
DSETOPS_FIFO_WRITER_TO_READER=fifo_dsetops_writer_to_reader
DSETOPS_FIFO_READER_TO_WRITER=fifo_dsetops_reader_to_writer

###############################################################################
## short hands and function definitions
###############################################################################
DPRINT=:              # Set to "echo Debug:" for debugging printing,
                        # else ":" for noop.
IFDEBUG=:             # Set to null to turn on debugging, else ":" for noop.

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Testing".
#
TESTING() {
   SPACES="                                                               "
   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
}

# To wait for the writer message file or till the maximum # of seconds is reached
# $1 is the message file to wait for
# This performs similar function as the routine h5_wait_message() in test/h5test.c
WAIT_MESSAGE() {
    message=$1                                  # Get the name of the message file to wait for
    t0=`date +%s`                               # Get current time in seconds
    difft=0                                     # Initialize the time difference
    mexist=0                                    # Indicate whether the message file is found
    while [ $difft -lt $MESSAGE_TIMEOUT ] ;     # Loop till message times out
    do
        t1=`date +%s`                           # Get current time in seconds
        difft=`expr $t1 - $t0`                  # Calculate the time difference
        if [ -e $message ]; then                # If message file is found:
            mexist=1                            #       indicate the message file is found
            rm $message                         #       remove the message file
            break                               #       get out of the while loop
        fi
    done;
    if test $mexist -eq 0; then
        # Issue warning that the writer message file is not found, continue with launching the reader(s)
        echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds
    else
        echo $WRITER_MESSAGE is found
    fi
}

###############################################################################
## Main
##
## Modifications:
##   Vailin Choi; July 2013
##     Add waiting of message file before launching the reader(s).
##     Due to the implementation of file locking, coordination
##     is needed in file opening for the writer/reader tests
##     to proceed as expected.
##
###############################################################################
# The build (current) directory might be different than the source directory.
if test -z "$srcdir"; then
   srcdir=.
fi

# Check to see if the VFD specified by the HDF5_DRIVER environment variable
# supports SWMR and/or if we are using parallel HDF5, which does not
# currently support SWMR.
./vfd_swmr_check_compat
rc=$?
if [ $rc -ne 0 ] ; then
    echo
    echo "Either the VFD specified by the HDF5_DRIVER environment variable"
    echo "does not support VFD SWMR or parallel HDF5 was configured, which"
    echo "currently does not support SWMR."
    echo
    echo "VFD SWMR acceptance tests skipped"
    echo
    exit 0
fi

all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups attrdset dsetops"
all_tests="${all_tests} groups_attrs os_groups_attrs groups_ops os_groups_ops few_big many_small"
tests=${all_tests}

if [ $# -gt 0 ]; then
	tests=
fi

for t; do
	if ! echo $all_tests | grep -q "\<${t}\>"; then
		echo "$t: Unknown test, ${t}"
		exit 1
	fi
	tests="${tests} ${t}"
done

echo tests=${tests}
for t in ${tests}; do
	eval do_${t}=yes
done

# HDF5 has several tests that create and delete signal files to communicate
# between processes, and it seems that even though the names of the files are
# different, occasionally the wrong file is deleted, interrupting the flow of
# the test.  Running each of these tests in its own directory should eliminate
# the problem.
rm -rf vfd_swmr_test
mkdir vfd_swmr_test

## With the --disable-shared option, swmr program files are built in the test
## directory, otherwise they are in test/.libs with a corresponding wrapper
## script in the test directory.  The programs or wrapper scripts in test should
## always be copied, swmr files in .libs should be copied only if they exists.
#if [ -f .libs/vfd_swmr ]; then
#    mkdir vfd_swmr_test/.libs
#    for FILE in .libs/vfd_swmr*; do
#        case "$FILE" in
#            *.o) continue ;;    ## don't copy the .o files
#        esac
#        cp $FILE vfd_swmr_test/.libs
#    done
#fi

cd vfd_swmr_test



# Loop over index types
for index_type in "-i ea" "-i b2" 
do
    # Try without compression, only; uncomment "-c 5" to try with compression.

    for compress in "" "-c 5"
    do
        echo
        echo "** Loop testing parameters: $index_type $compress"
        echo
	if [ ${do_generator:-no} = yes ]; then
		echo
		echo "## Generator test"
		# Launch the Generator without VFD SWMR write
		echo launch the vfd_swmr_generator
		../vfd_swmr_generator $compress $index_type
		if test $? -ne 0; then
		    echo generator had error
		    nerrors=`expr $nerrors + 1`
		fi

		# Launch the Generator with VFD SWMR write
		echo launch the vfd_swmr_generator with VFD SWMR write
		../vfd_swmr_generator -s $compress $index_type
		if test $? -ne 0; then
		    echo generator had error
		    nerrors=`expr $nerrors + 1`
		fi
	fi

	if [ ${do_expand:-no} = yes ]; then
		echo
		echo "## Writer test - test expanding the dataset"

		# Launch the Generator
		echo launch the vfd_swmr_generator with VFD SWMR write
		../vfd_swmr_generator -s $compress $index_type
		if test $? -ne 0; then
		    echo generator had error
		    nerrors=`expr $nerrors + 1`
		fi

		# Remove any possible writer message file before launching writer
		rm -f $WRITER_MESSAGE
		#
		# Launch the Writer
		echo launch the vfd_swmr_writer
		seed="" # Put -r <random seed> command here
		catch_out_err_and_rc vfd_swmr_writer \
		    ../vfd_swmr_writer -o $Nrecords $seed &
		pid_writer=$!
		$DPRINT pid_writer=$pid_writer

		# Wait for message from writer process before starting reader(s)
		WAIT_MESSAGE $WRITER_MESSAGE
		#
		# Launch the Readers
		#declare -a seeds=(<seed1> <seed2> <seed3> ... )
		echo launch $Nreaders vfd_swmr_readers
		pid_readers=""
		n=0
		while [ $n -lt $Nreaders ]; do
		    #seed="-r ${seeds[$n]}"
		    seed=""
		    catch_out_err_and_rc vfd_swmr_reader.$n \
			../vfd_swmr_reader $Nsecs_add $seed &
		    pid_readers="$pid_readers $!"
		    n=`expr $n + 1`
		done
		$DPRINT pid_readers=$pid_readers
		$IFDEBUG ps

		# Wait for the readers to finish before signalling the
		# writer to quit: the writer holds the file open so that the
		# readers will find the shadow file when they reopen
		# the .h5 file.
		wait $pid_readers
		kill -USR1 $(cat vfd_swmr_writer.pid)
		wait $pid_writer

		# Collect exit codes of the readers
		n=0
		while [ $n -lt $Nreaders ]; do
		    if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then
			echo reader had error
			nerrors=$((nerrors + 1))
		    fi
		    n=$((n + 1))
		done

		# Collect exit code of the writer
		$DPRINT checked writer $pid_writer
		if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then
		    echo writer had error
		    nerrors=$((nerrors + 1))
		fi

		# Clean up output files
		rm -f vfd_swmr_writer.{out,rc}
		rm -f vfd_swmr_reader.*.{out,rc}
	fi

	if [ ${do_shrink:-no} = yes ]; then
		if [ ${do_expand:-no} != yes ]; then
			echo "Cancelling the 'shrink' test: it depends on the .h5 file left behind by the 'expand' test." 1>&2
			exit 1
		fi
		echo
		echo "## Remove test - test shrinking the dataset"

		# Remove any possible writer message file before launching writer
		rm -f $WRITER_MESSAGE
		# Launch the Remove Writer
		echo launch the vfd_swmr_remove_writer
		seed="" # Put -r <random seed> command here
		catch_out_err_and_rc vfd_swmr_writer \
		    ../vfd_swmr_remove_writer -o $Nrecs_rem $seed &
		pid_writer=$!
		$DPRINT pid_writer=$pid_writer

		# Wait for message from writer process before starting reader(s)
		WAIT_MESSAGE $WRITER_MESSAGE
		#
		# Launch the Remove Readers
		#declare -a seeds=(<seed1> <seed2> <seed3> ... )
		n=0
		pid_readers=""
		echo launch $Nreaders swmr_remove_readers
		while [ $n -lt $Nreaders ]; do
		    #seed="-r ${seeds[$n]}"
		    seed=""
		    catch_out_err_and_rc vfd_swmr_reader.$n \
			../vfd_swmr_remove_reader $Nsecs_rem $seed &
		    pid_readers="$pid_readers $!"
		    n=`expr $n + 1`
		done
		$DPRINT pid_readers=$pid_readers
		$IFDEBUG ps

		# Wait for the readers to finish before signalling the
		# writer to quit: the writer holds the file open so that the
		# readers will find the shadow file when they reopen
		# the .h5 file.
		wait $pid_readers
		kill -USR1 $(cat vfd_swmr_writer.pid)
		wait $pid_writer

		# Collect exit codes of the readers
		n=0
		while [ $n -lt $Nreaders ]; do
		    if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then
			echo reader had error
			nerrors=$((nerrors + 1))
		    fi
		    n=$((n + 1))
		done

		# Collect exit code of the writer
		$DPRINT checked writer $pid_writer
		if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then
		    echo writer had error
		    nerrors=$((nerrors + 1))
		fi

		# Clean up output files
		rm -f vfd_swmr_writer.{out,rc}
		rm -f vfd_swmr_reader.*.{out,rc}
	fi

	if [ ${do_expand_shrink:-no} = yes ]; then
		echo
		echo "## Expand/shrink test - randomly grow or shrink the dataset"

		# Launch the Generator
		echo launch the vfd_swmr_generator with VFD SWMR write
		../vfd_swmr_generator -s $compress $index_type
		if test $? -ne 0; then
		    echo generator had error
		    nerrors=`expr $nerrors + 1`
		fi

		# Launch the Writer (not in parallel - just to rebuild the datasets)
		echo launch the vfd_swmr_writer
		seed="" # Put -r <random seed> command here
		../vfd_swmr_writer -W $Nrecords $seed
		if test $? -ne 0; then
		    echo writer had error
		    nerrors=`expr $nerrors + 1`
		fi

		# Remove any possible writer message file before launching writer
		rm -f $WRITER_MESSAGE
		#
		# Launch the Add/Remove Writer
		echo launch the vfd_swmr_addrem_writer
		seed="" # Put -r <random seed> command here
		catch_out_err_and_rc vfd_swmr_writer \
		    ../vfd_swmr_addrem_writer $Nrecords $seed &
		pid_writer=$!
		$DPRINT pid_writer=$pid_writer

		# Wait for message from writer process before starting reader(s)
		WAIT_MESSAGE $WRITER_MESSAGE
		#
		# Launch the Add/Remove Readers
		#declare -a seeds=(<seed1> <seed2> <seed3> ... )
		n=0
		pid_readers=""
		echo launch $Nreaders vfd_swmr_remove_readers
		while [ $n -lt $Nreaders ]; do
		    #seed="-r ${seeds[$n]}"
		    seed=""
		    catch_out_err_and_rc vfd_swmr_reader.$n \
			../vfd_swmr_remove_reader $Nsecs_addrem $seed &
		    pid_readers="$pid_readers $!"
		    n=`expr $n + 1`
		done
		$DPRINT pid_readers=$pid_readers
		$IFDEBUG ps

		# Wait for the readers to finish before signalling the
		# writer to quit: the writer holds the file open so that the
		# readers will find the shadow file when they reopen
		# the .h5 file.
		wait $pid_readers
		kill -USR1 $(cat vfd_swmr_writer.pid)
		wait $pid_writer

		# Collect exit codes of the readers
		n=0
		while [ $n -lt $Nreaders ]; do
		    if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then
			echo reader had error
			nerrors=$((nerrors + 1))
		    fi
		    n=$((n + 1))
		done

		# Collect exit code of the writer
		$DPRINT checked writer $pid_writer
		if [ ! -e vfd_swmr_writer.rc ] ||
		   [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then
		    echo writer had error
		    nerrors=$((nerrors + 1))
		fi

		# Clean up output files
		rm -f vfd_swmr_writer.{out,rc}
		rm -f vfd_swmr_reader.*.{out,rc}
	fi

	if [ ${do_sparse:-no} = yes ]; then
		echo
		echo "## Sparse writer test - write random dataset locations"

		# Launch the Generator
		# NOTE: Random seed is shared between readers and writers and is
		#       created by the generator.
		echo launch the vfd_swmr_generator with VFD SWMR write
		seed="" # Put -r <random seed> command here
		../vfd_swmr_generator -s $compress $index_type $seed
		if test $? -ne 0; then
		    echo generator had error
		    nerrors=`expr $nerrors + 1`
		fi

		# Remove any possible writer message file before launching writer
		rm -f $WRITER_MESSAGE
		# Launch the Sparse writer
		echo launch the vfd_swmr_sparse_writer
		catch_out_err_and_rc vfd_swmr_writer nice -n 20 \
		    ../vfd_swmr_sparse_writer $Nrecs_spa &
		pid_writer=$!
		$DPRINT pid_writer=$pid_writer

		# Wait for message from writer process before starting reader(s)
		WAIT_MESSAGE $WRITER_MESSAGE
		#
		# Launch the Sparse readers
		n=0
		pid_readers=""
		echo launch $Nrdrs_spa vfd_swmr_sparse_readers
		while [ $n -lt $Nrdrs_spa ]; do
		    # The sparse reader spits out a LOT of data so it's set to 'quiet'
		    catch_out_err_and_rc vfd_swmr_reader.$n \
			../vfd_swmr_sparse_reader -q $Nrecs_spa &
		    pid_readers="$pid_readers $!"
		    n=`expr $n + 1`
		done
		$DPRINT pid_readers=$pid_readers
		$IFDEBUG ps

		# Wait for the readers and the writer to finish.
		echo "pid_readers=$pid_readers"
		echo "pid_writer=$pid_writer"

		# Wait for the readers to finish before signalling the
		# writer to quit: the writer holds the file open so that the
		# readers will find the shadow file when they reopen
		# the .h5 file.
		wait $pid_readers
		kill -USR1 $(cat vfd_swmr_writer.pid)
		wait $pid_writer

		# Collect exit codes of the readers
		n=0
		while [ $n -lt $Nrdrs_spa ]; do
		    if [ $(cat vfd_swmr_reader.$n.rc) -ne 0 ]; then
			echo reader had error
			nerrors=$((nerrors + 1))
		    fi
		    n=$((n + 1))
		done

		# Collect exit code of the writer
		$DPRINT checked writer $pid_writer
		if [ $(cat vfd_swmr_writer.rc) -ne 0 ]; then
		    echo writer had error
		    nerrors=$((nerrors + 1))
		fi

		# Clean up output files
		rm -f vfd_swmr_writer.{out,rc}
		rm -f vfd_swmr_reader.*.{out,rc}
	fi
    done
done

#
# Test variable-length strings, expecting errors.
#
for ty in null oob; do

	if [ ${ty} = null ]; then
		[ ${do_vlstr_null:-no} = no ] && continue
		echo
		echo "## VL string 1 - expect to read NULL"
	else
		[ ${do_vlstr_oob:-no} = no ] && continue
		echo
		echo "## VL string 2 - expect out-of-bounds access"
	fi

	echo launch vfd_swmr_vlstr_writer
	catch_out_err_and_rc vfd_swmr_vlstr_writer \
	    ../vfd_swmr_vlstr_writer -n 500 -q -t ${ty} &
	pid_writer=$!

	# pause?

	catch_out_err_and_rc vfd_swmr_vlstr_reader \
	    ../vfd_swmr_vlstr_reader -n 500 -q -t ${ty} &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	kill -USR1 $(cat vfd_swmr_vlstr_writer.pid)
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_vlstr_reader.rc) -ne 0 ]; then
		echo reader had error
		nsofterrors=$((nsofterrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_vlstr_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_vlstr_writer.{out,rc}
	rm -f vfd_swmr_vlstr_reader.*.{out,rc}
done

#
# Make sure that a "zoo"---the variety of HDF5 object types---can be
# read and written by VFD SWMR.
#
if [ ${do_zoo:-no} = yes ]; then
	rm -f ./shared_tick_num
	echo launch vfd_swmr_zoo_writer
	catch_out_err_and_rc vfd_swmr_zoo_writer \
	    ../vfd_swmr_zoo_writer -q &
	pid_writer=$!

        # -l is the expected maximal number of ticks from the writer's finishing zoo creation or deletion
        # to the reader's finishing validation of zoo creation or deletion
	catch_out_err_and_rc vfd_swmr_zoo_reader \
	    ../vfd_swmr_zoo_reader -l 4 -q &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_zoo_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_zoo_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_zoo_writer.{out,rc}
	rm -f vfd_swmr_zoo_reader.*.{out,rc}
fi

# attrdset test
for options in "-p -g -a 10 -v -m -d 10 -c 3 -u 5" "-k -a 20 -v -m -d 5"; do
    #
    # Test a few big datasets of one and two dimensions.
    #
    if [ ${do_attrdset:-no} = no ]; then
        continue
    fi

    # Clean up any existing fifo files from previous runs
    if [ -e ./$ATTRDSET_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
        rm -f ./$ATTRDSET_FIFO_WRITER_TO_READER
    fi
    if [ -e ./$ATTRDSET_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
        rm -f ./$ATTRDSET_FIFO_READER_TO_WRITER
    fi

    echo launch vfd_swmr_attrdset_writer attrdset, options $options
    catch_out_err_and_rc vfd_swmr_attrdset_writer \
        ../vfd_swmr_attrdset_writer $options &
    pid_writer=$!

    catch_out_err_and_rc vfd_swmr_attrdset_reader \
        ../vfd_swmr_attrdset_reader $options &
    pid_reader=$!

    # Wait for the reader to finish before signaling the
    # writer to quit: the writer holds the file open so that the
    # reader will find the shadow file when it opens
    # the .h5 file.
    wait $pid_reader
    wait $pid_writer

    # Collect exit code of the reader
    if [ $(cat vfd_swmr_attrdset_reader.rc) -ne 0 ]; then
        echo reader had error
        nerrors=$((nerrors + 1))
    fi

    # Collect exit code of the writer
    if [ $(cat vfd_swmr_attrdset_writer.rc) -ne 0 ]; then
        echo writer had error
        nerrors=$((nerrors + 1))
    fi

    # Clean up output files
    rm -f vfd_swmr_attrdset_writer.{out,rc}
    rm -f vfd_swmr_attrdset_reader.*.{out,rc}
done

# dsetops test
for options in "-p -e 20 -t" "-g -m 5 -n 2 -s 10 -w 7" "-k -m 10 -n 5 -r 5 -l 10"; do
	#
	#
	if [ ${do_dsetops:-no} = no ]; then
		continue
	fi
    # Clean up any existing fifo files from previous runs
    if [ -e ./$DSETOPS_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
        rm -f ./$DSETOPS_FIFO_WRITER_TO_READER
    fi
    if [ -e ./$DSETOPS_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
        rm -f ./$DSETOPS_FIFO_READER_TO_WRITER
    fi
    #
	echo launch vfd_swmr_dsetops_writer dsetops, options $options
	catch_out_err_and_rc vfd_swmr_dsetops_writer \
	    ../vfd_swmr_dsetops_writer $options &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_dsetops_reader \
	    ../vfd_swmr_dsetops_reader $options &
	pid_reader=$!

	# Wait for the reader to finish before signaling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_dsetops_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_dsetops_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_dsetops_writer.{out,rc}
	rm -f vfd_swmr_dsetops_reader.*.{out,rc}
done

#
# Make sure that we can create GROUP_n groups (20, 40, or 400 depending on the HDF5TestExpress level)
# while a reader waits for each to appear.
#
if [ ${do_groups:-no} = yes ]; then
	echo launch vfd_swmr_group_writer
	catch_out_err_and_rc vfd_swmr_group_writer \
	    ../vfd_swmr_group_writer -q -c 10 -n $GROUP_n &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_group_reader \
	    ../vfd_swmr_group_reader -q -c 10 -n $GROUP_n -u 5 &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_group_writer.{out,rc}
	rm -f vfd_swmr_group_reader.*.{out,rc}
fi

# The group attribute test takes longer. 
# So for standard run and quick run, we
# shorten the number of tests. The standard
# run covers all the features we need to 
# test. The quick run doesn't cover the
# attribute storage change between dense and 
# compact.
# The exhaustive run tries to test a feature
# per test from scratch.
# 
grp_attr_list=(
                "compact"
                "dense"
                "compact-del"
                "dense-del"
                "compact-add-to-dense"
                "dense-del-to-compact"
                "modify"
                "add-vstr"
                "remove-vstr"
                "modify-vstr"
                "add-ohr-block"
                "del-ohr-block"
               )
grp_sub_attr_list=(
                "dense-del-to-compact"
                "modify"
                "remove-vstr"
                "modify-vstr"
                "del-ohr-block"
               )

grp_short_sub_attr_list=(
                "dense"
                "modify"
                "remove-vstr"
                "modify-vstr"
                "del-ohr-block"
               )

if [[ "$HDF5TestExpress" -eq 1 ]] ; then       #Setting for standard run
    grp_attr_list=("${grp_sub_attr_list[@]}")
elif [[ "$HDF5TestExpress" -gt 1 ]] ; then     #Setting for quick run
    grp_attr_list=("${grp_short_sub_attr_list[@]}")
fi

for options in ${grp_attr_list[*]}; do
	if [ ${do_groups_attrs:-no} = no ]; then
		continue
	fi
	echo launch vfd_swmr_group attribute: $options
	catch_out_err_and_rc vfd_swmr_group_writer \
	    ../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_group_reader \
	    ../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_group_writer.{out,rc}
	rm -f vfd_swmr_group_reader.*.{out,rc}
done

# The following tests are for add/del/modify attributes for
# groups created with the old-style. 
# Check https://portal.hdfgroup.org/display/HDF5/Groups for 
# the detailed group implementation note. 
# The 'compact' and 'compact-del' are the attribute addition
# and deletion tests. Other test names have the same meaning 
# as those of the new-style group tests.
# 
os_grp_attr_list=(
                "compact"
                "compact-del"
                "modify"
                "add-vstr"
                "remove-vstr"
                "modify-vstr"
                "add-ohr-block"
                "del-ohr-block"
 )
os_grp_sub_attr_list=(
                "modify"
                "remove-vstr"
                "modify-vstr"
                "del-ohr-block"
               )
if [[ "$HDF5TestExpress" -gt 0 ]] ; then       #Setting for standard run
    os_grp_attr_list=("${os_grp_sub_attr_list[@]}")
fi

for options in ${os_grp_attr_list[*]}; do
	if [ ${do_os_groups_attrs:-no} = no ]; then
		continue
	fi
	echo launch vfd_swmr_group attribute with old-style group: $options
	catch_out_err_and_rc vfd_swmr_group_writer \
	    ../vfd_swmr_group_writer -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_group_reader \
	    ../vfd_swmr_group_reader -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_group_writer.{out,rc}
	rm -f vfd_swmr_group_reader.*.{out,rc}
done

# The group operation test takes longer. 
# So for the quick run, we
# shorten the number of tests. 
# The essential features are covered. 
grp_op_list=(
                "grp-creation"
                "grp-deletion"
                "grp-move"
                "grp-ins-links"
                "grp-del-links"
                "grp-compact-t-dense"
                "grp-dense-t-compact"
               )
grp_sub_op_list=(
                "grp-move"
                "grp-dense-t-compact"
               )


if [[ "$HDF5TestExpress" -gt 1 ]] ; then       #Setting for express run
    grp_op_list=("${grp_sub_op_list[@]}")
fi

for options in ${grp_op_list[*]}; do
	if [ ${do_groups_ops:-no} = no ]; then
		continue
	fi
	echo launch vfd_swmr_group operations: $options
	catch_out_err_and_rc vfd_swmr_group_writer \
	    ../vfd_swmr_group_writer -q -c 1 -n $GROUP_op_n -O $options &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_group_reader \
	    ../vfd_swmr_group_reader -q -c 1 -n $GROUP_op_n -O $options &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_group_writer.{out,rc}
	rm -f vfd_swmr_group_reader.*.{out,rc}
done

# Test the group operations for old-style
os_grp_op_list=(
                "grp-creation"
                "grp-deletion"
                "grp-move"
                "grp-ins-links"
                "grp-del-links"
               )
os_grp_sub_op_list=(
                "grp-move"
                "grp-ins-links"
                "grp-del-links"
               )


if [[ "$HDF5TestExpress" -gt 1 ]] ; then       #Setting for express run
    os_grp_op_list=("${os_grp_sub_op_list[@]}")
fi

for options in ${os_grp_op_list[*]}; do
	if [ ${do_os_groups_ops:-no} = no ]; then
		continue
	fi
	echo launch vfd_swmr_group operations with old-style group: $options
	catch_out_err_and_rc vfd_swmr_group_writer \
	    ../vfd_swmr_group_writer -q -G -c 1 -n $GROUP_op_n -O $options &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_group_reader \
	    ../vfd_swmr_group_reader -q -G -c 1 -n $GROUP_op_n -O $options &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_group_writer.{out,rc}
	rm -f vfd_swmr_group_reader.*.{out,rc}
done



for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
	if [ ${do_many_small:-no} = no ]; then
		continue
	fi
	#
	# Test many small datasets of one and two dimensions.
	#
	# Perform 50 iterations on 1000 extensible datasets configured with
	# 16x16 chunks of 32-bit unsigned integer elements,
	# expanding each dataset by a chunk in one dimension (up to 50x1
	# 16x16 chunks) on each iteration.
	#
	# Perform the test again, extending each dataset
	# in *two* dimensions (up to 50x50 16x16 chunks).
	#
	echo launch vfd_swmr_bigset_writer many small, options $options
	catch_out_err_and_rc vfd_swmr_bigset_writer \
	    ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_bigset_reader \
	    ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -W &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	kill -USR1 $(cat vfd_swmr_bigset_writer.pid)
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_bigset_writer.{out,rc}
	rm -f vfd_swmr_bigset_reader.*.{out,rc}
done

for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
	#
	# Test a few big datasets of one and two dimensions.
	#
	# Perform 50 iterations on 5 extensible datasets configured with
	# 256x256 chunks of 32-bit unsigned integer elements,
	# expanding each dataset by a chunk in one dimension (up to 50x1
	# 256x256 chunks) on each iteration.
	#
	# Perform the test again, extending each dataset
	# in *two* dimensions (up to 50x50 256x256 chunks).
	#
	if [ ${do_few_big:-no} = no ]; then
		continue
	fi
	echo launch vfd_swmr_bigset_writer few big, options $options
	catch_out_err_and_rc vfd_swmr_bigset_writer \
	    ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q &
	pid_writer=$!

	catch_out_err_and_rc vfd_swmr_bigset_reader \
	    ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -W &
	pid_reader=$!

	# Wait for the reader to finish before signalling the
	# writer to quit: the writer holds the file open so that the
	# reader will find the shadow file when it opens
	# the .h5 file.
	wait $pid_reader
	kill -USR1 $(cat vfd_swmr_bigset_writer.pid)
	wait $pid_writer

	# Collect exit code of the reader
	if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then
		echo reader had error
		nerrors=$((nerrors + 1))
	fi

	# Collect exit code of the writer
	if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then
	    echo writer had error
	    nerrors=$((nerrors + 1))
	fi

	# Clean up output files
	rm -f vfd_swmr_bigset_writer.{out,rc}
	rm -f vfd_swmr_bigset_reader.*.{out,rc}
done

###############################################################################
## Report and exit
###############################################################################
cd ..
$DPRINT nerrors $nerrors nsofterrors $nsofterrors
if test $nerrors -eq 0 ; then
    echo "VFD SWMR tests passed."
    if test $nsofterrors -ne 0 ; then
	echo
        echo "${nsofterrors} expected errors occurred.  Expected errors are ok."
    fi
    if test -z "$HDF5_NOCLEANUP"; then
        # delete the test directory
        rm -rf vfd_swmr_test
    fi
    exit 0
else
    echo -n "VFD SWMR tests failed with $nerrors unexpected errors "
    echo "and $nsofterrors expected errors.  Expected errors are ok."
    echo "Please report unexpected errors, they may indicate a bug."
    exit 1
fi