philipp/hdf5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make the initial bug fixes to allow >2GB writes with Independent IO

Browse Source
This commit is contained in:
Richard Warren
2019-10-05 10:18:17 -04:00
parent f2be6da714
commit 21a18a1cd2
4 changed files with 47 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/H5FDmpio.c
View File

@@ -22,16 +22,17 @@
#include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */
#include "H5private.h" /* Generic Functions */
#include "H5private.h" /* Generic Functions */
#include "H5CXprivate.h" /* API Contexts */
#include "H5Dprivate.h" /* Dataset functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fprivate.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDmpi.h" /* MPI-based file drivers */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Dprivate.h" /* Dataset functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fprivate.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDmpi.h" /* MPI-based file drivers */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
#include "H5Sprivate.h" /* Large MPIO data type creation */
#ifdef H5_HAVE_PARALLEL
@@ -1324,6 +1325,7 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__mpio_read() */
/*-------------------------------------------------------------------------
* Function: H5FD__mpio_write
@@ -1366,6 +1368,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
#endif
int size_i;
hbool_t use_view_this_time = FALSE;
hbool_t derived_type = FALSE;
H5FD_mpio_xfer_t xfer_mode; /* I/O transfer mode */
herr_t ret_value = SUCCEED;
@@ -1391,8 +1394,6 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
if(H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off) < 0)
HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off")
size_i = (int)size;
if((hsize_t)size_i != size)
HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from size to size_i")
#ifdef H5FDmpio_DEBUG
if(H5FD_mpio_Debug[(int)'w'])
@@ -1430,6 +1431,20 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
*/
mpi_off = 0;
} /* end if */
else if(size != (hsize_t)size_i) {
/* If HERE, then we need to work around the integer size limit
* of 2GB. The input size_t size variable cannot fit into an integer,
* but we can get around that limitation by creating a different datatype
* and then setting the integer size (or element count) to 1 when using
* the derived_type.
*/
if (H5S_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0)
HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype")
derived_type = TRUE;
size_i = 1;
}
/* Write the data. */
if(use_view_this_time) {
@@ -1506,6 +1521,9 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
file->local_eof = addr + (haddr_t)bytes_written;
done:
if(derived_type) {
MPI_Type_free(&buf_type);
}
#ifdef H5FDmpio_DEBUG
if(H5FD_mpio_Debug[(int)'t'])
HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value );

27
src/H5Smpio.c
View File

@@ -34,6 +34,7 @@
#include "H5FLprivate.h" /* Free Lists */
#include "H5MMprivate.h" /* Memory management */
#include "H5Spkg.h" /* Dataspaces */
#include "H5Sprivate.h" /* H5S prototypes, etc. */
#include "H5VMprivate.h" /* Vector and array functions */
#ifdef H5_HAVE_PARALLEL
@@ -44,7 +45,7 @@
#define H5S_MPIO_INITIAL_ALLOC_COUNT 256
#define TWO_GIG_LIMIT 2147483648
#ifndef H5S_MAX_MPI_COUNT
#define H5S_MAX_MPI_COUNT 536870911 /* (2^29)-1 */
#define H5S_MAX_MPI_COUNT 1073741824
#endif
@@ -88,8 +89,6 @@ static herr_t H5S__release_datatype(H5S_mpio_mpitype_list_t *type_list);
static herr_t H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down,
size_t elmt_size, const MPI_Datatype *elmt_type, MPI_Datatype *span_type,
H5S_mpio_mpitype_list_t *type_list, uint64_t op_gen);
static herr_t H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
MPI_Datatype old_type, MPI_Datatype *new_type);
/*****************************/
@@ -183,7 +182,7 @@ H5S__mpio_all_type(const H5S_t *space, size_t elmt_size,
} /* end if */
else {
/* Create a LARGE derived datatype for this transfer */
if(H5S__mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0)
if(H5S_mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large datatype from the all selection")
*count = 1;
*is_derived_type = TRUE;
@@ -824,7 +823,7 @@ if(H5DEBUG(S)) {
} /* end if */
else
/* Create the compound datatype for this operation (> 2GB) */
if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0)
if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large inner datatype in hyper selection")
/*******************************************************
@@ -878,7 +877,7 @@ if(H5DEBUG(S))
* Again we need to check that the number of BLOCKS can fit into
* a 32 bit integer */
if(bigio_count < d[i].block) {
if(H5S__mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0)
if(H5S_mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large block datatype in hyper selection")
} /* end if */
else
@@ -899,7 +898,7 @@ if(H5DEBUG(S))
* we call the large type creation function to handle that
*/
if(bigio_count < d[i].count) {
if(H5S__mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0)
if(H5S_mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large outer datatype in hyper selection")
} /* end if */
/* otherwise a regular create_hvector will do */
@@ -1020,7 +1019,7 @@ H5S__mpio_span_hyper_type(const H5S_t *space, size_t elmt_size,
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
} /* end if */
else
if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0)
if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection")
elmt_type_is_derived = TRUE;
@@ -1203,7 +1202,7 @@ H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down,
/* create the block type from elmt_type while checking the 32 bit int limit */
if(blocklen[u] > bigio_count) {
if(H5S__mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0)
if(H5S_mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection")
} /* end if */
else
@@ -1455,7 +1454,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5S__mpio_create_large_type
* Function: H5S_mpio_create_large_type
*
* Purpose: Create a large datatype of size larger than what a 32 bit integer
* can hold.
@@ -1468,8 +1467,8 @@ done:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
herr_t
H5S_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
MPI_Datatype old_type, MPI_Datatype *new_type)
{
int num_big_types; /* num times the 2G datatype will be repeated */
@@ -1481,7 +1480,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
MPI_Aint disp[2], old_extent;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
FUNC_ENTER_NOAPI_NOINIT
/* Calculate how many Big MPI datatypes are needed to represent the buffer */
num_big_types = (int)(num_elements/bigio_count);
@@ -1559,7 +1558,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5S__mpio_create_large_type() */
} /* end H5S_mpio_create_large_type() */
#endif /* H5_HAVE_PARALLEL */

5
src/H5Sprivate.h
View File

@@ -308,6 +308,11 @@ H5_DLL herr_t H5S_sel_iter_close(H5S_sel_iter_t *sel_iter);
#ifdef H5_HAVE_PARALLEL
H5_DLL hsize_t H5S_mpio_set_bigio_count(hsize_t new_count);
H5_DLL herr_t H5S_mpio_create_large_type(hsize_t num_elements,
MPI_Aint stride_bytes,
MPI_Datatype old_type,
MPI_Datatype *new_type);
H5_DLL herr_t H5S_mpio_space_type(const H5S_t *space, size_t elmt_size,
/* out: */ MPI_Datatype *new_type,
int *count,

3
testpar/t_file.c
View File

@@ -145,7 +145,7 @@ test_page_buffer_access(void)
ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0);
VRFY((ret == 0), "");
ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100);
ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*128);
VRFY((ret == 0), "");
ret = H5Pset_page_buffer_size(fapl, sizeof(int)*100000, 0, 0);
VRFY((ret == 0), "");
@@ -180,7 +180,6 @@ test_page_buffer_access(void)
data[i] = -1;
if(MAINPROCESS) {
hid_t fapl_self = H5I_INVALID_HID;
fapl_self = create_faccess_plist(MPI_COMM_SELF, MPI_INFO_NULL, facc_type);
ret = H5Pset_page_buffer_size(fapl_self, sizeof(int)*1000, 0, 0);