philipp/hdf5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored testpar/t_bigio.c and then added a basic description of these test changes into RELEASE.txt (HDFFV-10301)

Browse Source
This commit is contained in:
Richard.Warren
2017-11-08 15:31:03 -06:00
parent 2184928e8b
commit 51a5ef118c
2 changed files with 95 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
release_docs/RELEASE.txt
View File

@@ -287,6 +287,16 @@ Bug Fixes since HDF5-1.10.1 release
(HDFFV-7853, VC, 2017/07/05)
- Refactored the testpar/t_bigio.c test to include ALARM macros
Changed the test to include the ALARM_ON and ALARM_OFF macros which
are intended to prevent nightly test hangs that have been observed
with this particular parallel test example. The code was also modified to
simplify status reporting (only from MPI rank 0) and additional
status checking added.
(RAW - 2017/11/08, HDFFV-10301)
Configuration
-------------
- cmake MPI

197
testpar/t_bigio.c
View File

@@ -1,9 +1,12 @@
#include "hdf5.h"
#include "testpar.h"
#include "testphdf5.h"
#include "H5Dprivate.h" /* For Chunk tests */
// int TestVerbosity = VERBO_LO; /* Default Verbosity is Low */
/* FILENAME and filenames must have the same number of names */
const char *FILENAME[2]={ "bigio_test.h5",
NULL
};
/* Constants definitions */
#define MAX_ERR_REPORT 10 /* Maximum number of errors reported */
@@ -38,7 +41,6 @@ typedef hsize_t B_DATATYPE;
int facc_type = FACC_MPIO; /*Test file access type */
int dxfer_coll_type = DXFER_COLLECTIVE_IO;
size_t bigcount = DXFER_BIGCOUNT;
char filename[20] = "bigio_test.h5";
int nerrors = 0;
int mpi_size, mpi_rank;
@@ -506,7 +508,7 @@ dataset_big_write(void)
H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL);
/* create the file collectively */
fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl);
fid = H5Fcreate(FILENAME[0], H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl);
VRFY((fid >= 0), "H5Fcreate succeeded");
/* Release file-access template */
@@ -515,7 +517,8 @@ dataset_big_write(void)
/* Each process takes a slabs of rows. */
printf("\nTesting Dataset1 write by ROW\n");
if (MAINPROCESS)
HDprintf("\nTesting Dataset1 write by ROW\n");
/* Create a large dataset */
dims[0] = bigcount;
dims[1] = mpi_size;
@@ -563,21 +566,6 @@ dataset_big_write(void)
VRFY((ret>= 0),"set independent IO collectively succeeded");
}
/* write data collectively */
MESG("writeAll by Row");
{
int j,k =0;
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", wdata[k]);
k++;
}
}
}
printf("\n");
}
ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace,
xfer_plist, wdata);
VRFY((ret >= 0), "H5Dwrite dataset1 succeeded");
@@ -591,9 +579,9 @@ dataset_big_write(void)
VRFY((ret >= 0), "H5Dclose1 succeeded");
/* Each process takes a slabs of cols. */
printf("\nTesting Dataset2 write by COL\n");
if (MAINPROCESS)
HDprintf("\nTesting Dataset2 write by COL\n");
/* Create a large dataset */
dims[0] = bigcount;
dims[1] = mpi_size;
@@ -641,21 +629,6 @@ dataset_big_write(void)
VRFY((ret>= 0),"set independent IO collectively succeeded");
}
/* write data collectively */
MESG("writeAll by Col");
{
int j,k =0;
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", wdata[k]);
k++;
}
}
}
printf("\n");
}
ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace,
xfer_plist, wdata);
VRFY((ret >= 0), "H5Dwrite dataset1 succeeded");
@@ -671,7 +644,8 @@ dataset_big_write(void)
/* ALL selection */
printf("\nTesting Dataset3 write select ALL proc 0, NONE others\n");
if (MAINPROCESS)
HDprintf("\nTesting Dataset3 write select ALL proc 0, NONE others\n");
/* Create a large dataset */
dims[0] = bigcount;
dims[1] = 1;
@@ -719,21 +693,6 @@ dataset_big_write(void)
MESG("data_array created");
}
/* write data collectively */
MESG("writeAll by process 0");
{
int j,k =0;
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", wdata[k]);
k++;
}
}
}
printf("\n");
}
ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace,
xfer_plist, wdata);
VRFY((ret >= 0), "H5Dwrite dataset1 succeeded");
@@ -747,7 +706,8 @@ dataset_big_write(void)
VRFY((ret >= 0), "H5Dclose1 succeeded");
/* Point selection */
printf("\nTesting Dataset4 write point selection\n");
if (MAINPROCESS)
HDprintf("\nTesting Dataset4 write point selection\n");
/* Create a large dataset */
dims[0] = bigcount;
dims[1] = mpi_size * 4;
@@ -822,12 +782,14 @@ dataset_big_write(void)
/* Need larger memory for data buffer */
free(wdata);
#if 0
/* Ridiculus test... */
wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE));
VRFY((wdata != NULL), "wdata malloc succeeded");
printf("\nTesting Dataset5 write irregular selection\n");
if (MAINPROCESS)
HDprintf("\nTesting Dataset5 write irregular selection\n");
/* Create a large dataset */
dims[0] = bigcount/6;
dims[0] = bigcount/mpi_size;
dims[1] = mpi_size * 4;
sid = H5Screate_simple (RANK, dims, NULL);
@@ -881,7 +843,8 @@ dataset_big_write(void)
ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_OR, start, stride, count, block);
VRFY((ret >= 0), "H5Sset_hyperslab succeeded");
}
printf("Setting up for collective transfer\n");
if (MAINPROCESS)
HDprintf("Setting up for collective transfer\n");
/* set up the collective transfer properties list */
xfer_plist = H5Pcreate (H5P_DATASET_XFER);
VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded");
@@ -960,15 +923,16 @@ dataset_big_read(void)
H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL);
/* open the file collectively */
fid=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl);
fid=H5Fopen(FILENAME[0],H5F_ACC_RDONLY,acc_tpl);
VRFY((fid >= 0), "H5Fopen succeeded");
/* Release file-access template */
ret = H5Pclose(acc_tpl);
VRFY((ret >= 0), "");
if (MAINPROCESS)
HDprintf("\nRead Testing Dataset1 by COL\n");
printf("\nRead Testing Dataset1 by COL\n");
dataset = H5Dopen2(fid, DATASET1, H5P_DEFAULT);
VRFY((dataset >= 0), "H5Dopen2 succeeded");
@@ -1016,18 +980,6 @@ dataset_big_read(void)
xfer_plist, rdata);
VRFY((ret >= 0), "H5Dread dataset1 succeeded");
{
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", rdata[k]);
k++;
}
}
}
printf("\n");
}
/* verify the read data with original expected data */
ret = verify_data(start, count, stride, block, rdata, wdata);
if(ret) {fprintf(stderr, "verify failed\n"); exit(1);}
@@ -1040,7 +992,8 @@ dataset_big_read(void)
VRFY((ret >= 0), "H5Dclose1 succeeded");
printf("\nRead Testing Dataset2 by ROW\n");
if (MAINPROCESS)
HDprintf("\nRead Testing Dataset2 by ROW\n");
memset(rdata, 0, bigcount*sizeof(B_DATATYPE));
dataset = H5Dopen2(fid, DATASET2, H5P_DEFAULT);
VRFY((dataset >= 0), "H5Dopen2 succeeded");
@@ -1089,18 +1042,6 @@ dataset_big_read(void)
xfer_plist, rdata);
VRFY((ret >= 0), "H5Dread dataset2 succeeded");
{
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", rdata[k]);
k++;
}
}
}
printf("\n");
}
/* verify the read data with original expected data */
ret = verify_data(start, count, stride, block, rdata, wdata);
if(ret) {fprintf(stderr, "verify failed\n"); exit(1);}
@@ -1112,8 +1053,8 @@ dataset_big_read(void)
ret = H5Dclose(dataset);
VRFY((ret >= 0), "H5Dclose1 succeeded");
printf("\nRead Testing Dataset3 read select ALL proc 0, NONE others\n");
if (MAINPROCESS)
HDprintf("\nRead Testing Dataset3 read select ALL proc 0, NONE others\n");
memset(rdata, 0, bigcount*sizeof(B_DATATYPE));
dataset = H5Dopen2(fid, DATASET3, H5P_DEFAULT);
VRFY((dataset >= 0), "H5Dopen2 succeeded");
@@ -1163,18 +1104,6 @@ dataset_big_read(void)
xfer_plist, rdata);
VRFY((ret >= 0), "H5Dread dataset3 succeeded");
{
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(k < 10) {
printf("%lld ", rdata[k]);
k++;
}
}
}
printf("\n");
}
if(MAINPROCESS) {
/* verify the read data with original expected data */
ret = verify_data(start, count, stride, block, rdata, wdata);
@@ -1188,7 +1117,8 @@ dataset_big_read(void)
ret = H5Dclose(dataset);
VRFY((ret >= 0), "H5Dclose1 succeeded");
printf("\nRead Testing Dataset4 with Point selection\n");
if (MAINPROCESS)
HDprintf("\nRead Testing Dataset4 with Point selection\n");
dataset = H5Dopen2(fid, DATASET4, H5P_DEFAULT);
VRFY((dataset >= 0), "H5Dopen2 succeeded");
@@ -1258,11 +1188,24 @@ dataset_big_read(void)
ret = H5Dclose(dataset);
VRFY((ret >= 0), "H5Dclose1 succeeded");
printf("\nRead Testing Dataset5 with Irregular selection\n");
/* Need larger memory for data buffer */
free(wdata);
free(rdata);
wdata = NULL;
rdata = NULL;
/* We never wrote Dataset5 in the write section, so we can't
* expect to read it...
*/
file_dataspace = -1;
mem_dataspace = -1;
xfer_plist = -1;
dataset = -1;
#if 0
/* Need larger memory for data buffer */
if (MAINPROCESS)
HDprintf("\nRead Testing Dataset5 with Irregular selection\n");
wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE));
VRFY((wdata != NULL), "wdata malloc succeeded");
rdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE));
@@ -1369,20 +1312,22 @@ dataset_big_read(void)
ret = verify_data(start, count, stride, block, rdata, wdata);
if(ret) {fprintf(stderr, "verify failed\n"); exit(1);}
}
#endif
/* release all temporary handles. */
H5Sclose(file_dataspace);
H5Sclose(mem_dataspace);
H5Pclose(xfer_plist);
ret = H5Dclose(dataset);
VRFY((ret >= 0), "H5Dclose1 succeeded");
if (file_dataspace != -1) H5Sclose(file_dataspace);
if (mem_dataspace != -1) H5Sclose(mem_dataspace);
if (xfer_plist != -1) H5Pclose(xfer_plist);
if (dataset != -1) {
ret = H5Dclose(dataset);
VRFY((ret >= 0), "H5Dclose1 succeeded");
}
H5Fclose(fid);
/* release data buffers */
if(rdata) free(rdata);
if(wdata) free(wdata);
#endif
} /* dataset_large_readAll */
@@ -1478,6 +1423,7 @@ create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type)
void
coll_chunk1(void)
{
const char *filename = FILENAME[0];
if (MAINPROCESS)
printf("coll_chunk1\n");
@@ -1531,6 +1477,7 @@ coll_chunk1(void)
void
coll_chunk2(void)
{
const char *filename = FILENAME[0];
if (MAINPROCESS)
printf("coll_chunk2\n");
@@ -1585,8 +1532,9 @@ coll_chunk2(void)
void
coll_chunk3(void)
{
const char *filename = FILENAME[0];
if (MAINPROCESS)
printf("coll_chunk3\n");
printf("coll_chunk3\n");
coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER);
coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, POINT, OUT_OF_ORDER);
@@ -1956,7 +1904,7 @@ coll_chunktest(const char* filename,
acc_plist = create_faccess_plist(comm, info, facc_type);
VRFY((acc_plist >= 0),"MPIO creation property list succeeded");
file = H5Fopen(filename,H5F_ACC_RDONLY,acc_plist);
file = H5Fopen(FILENAME[0],H5F_ACC_RDONLY,acc_plist);
VRFY((file >= 0),"H5Fcreate succeeded");
status = H5Pclose(acc_plist);
@@ -2134,16 +2082,34 @@ int main(int argc, char **argv)
{
int ExpressMode = 0;
hsize_t newsize = 1048576;
/* Set the bigio processing limit to be 'newsize' bytes */
hsize_t oldsize = H5S_mpio_set_bigio_count(newsize);
/* Having set the bigio handling to a size that is managable,
* we'll set our 'bigcount' variable to be 2X that limit so
* that we try to ensure that our bigio handling is actually
* envoked and tested.
*/
if (newsize != oldsize) {
bigcount = newsize * 2;
bigcount = newsize * 2;
}
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
/* Attempt to turn off atexit post processing so that in case errors
* happen during the test and the process is aborted, it will not get
* hang in the atexit post processing in which it may try to make MPI
* calls. By then, MPI calls may not work.
*/
if (H5dont_atexit() < 0){
HDprintf("Failed to turn off atexit processing. Continue.\n");
};
/* set alarm. */
ALARM_ON;
ExpressMode = do_express_test(mpi_rank);
dataset_big_write();
@@ -2153,7 +2119,8 @@ int main(int argc, char **argv)
MPI_Barrier(MPI_COMM_WORLD);
if (ExpressMode > 0) {
printf("***Express test mode on. Several tests are skipped\n");
if (MAINPROCESS)
HDprintf("***Express test mode on. Several tests are skipped\n");
}
else {
coll_chunk1();
@@ -2163,6 +2130,12 @@ int main(int argc, char **argv)
coll_chunk3();
}
/* turn off alarm */
ALARM_OFF;
if (MAINPROCESS)
HDremove(FILENAME[0]);
/* close HDF5 library */
H5close();