philipp/hdf5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'vfd_swmr_log' into feature/vfd_swmr

Browse Source
This commit is contained in:
myang6
2021-10-28 10:22:10 -05:00
parent 572201b560 645aadd77b
commit caec2d284f
9 changed files with 3140 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/H5FDvfd_swmr_private.h
View File

@@ -84,4 +84,13 @@ H5_DLL herr_t H5F_vfd_swmr_insert_entry_eot(struct H5F_t *f);
H5_DLL void H5F_vfd_swmr_update_entry_eot(eot_queue_entry_t *);
H5_DLL herr_t H5F_dump_eot_queue(void);
/***************************************/
/* Log Macros and Functions */
/***************************************/
/* VFD SWMR Helper macros to calcuate the elapsed time */
/* The total time is in seconds */
#define TOTAL_TIME_PASSED(X, Y) \
((double)((Y.tv_sec - X.tv_sec) * 1000000000 + (Y.tv_nsec - X.tv_nsec))) / 1000000000.0
#endif /* H5FDvfd_swmr_private_H */

21
src/H5Fint.c
View File

@@ -2013,7 +2013,23 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Short cuts */
shared = file->shared;
lf = shared->lf;
/* Set up the VFD SWMR LOG file */
/* Kent*/
if (vfd_swmr_config_ptr->version) {
if (HDstrlen(vfd_swmr_config_ptr->log_file_path) > 0)
shared->vfd_swmr_log_on = TRUE;
if (TRUE == shared->vfd_swmr_log_on) {
/* Create the log file */
if ((shared->vfd_swmr_log_file_ptr = HDfopen(vfd_swmr_config_ptr->log_file_path, "w")) == NULL)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create the log file")
if (HDclock_gettime(CLOCK_MONOTONIC, &(shared->vfd_swmr_log_start_time)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get time via clock_gettime");
}
}
/* End of Kent */
lf = shared->lf;
/* Set the file locking flag. If the file is already open, the file
* requested file locking flag must match that of the open file.
@@ -2210,6 +2226,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
/* Success */
ret_value = file;
#if 1 /*Kent: write to the log file when H5F_open ends. Tested, can be commented out if necessary.*/
H5F_POST_VFD_SWMR_LOG_ENTRY(file, 1, "File open ends");
#endif
done:
if ((NULL == ret_value) && file) {
if (file->shared->root_grp && file->shared->nrefs == 1) {

52
src/H5Fpkg.h
View File

@@ -469,6 +469,10 @@ struct H5F_shared_t {
* manager
*/
FILE * vfd_swmr_log_file_ptr;
hbool_t vfd_swmr_log_on;
struct timespec vfd_swmr_log_start_time;
/* Delayed free space release doubly linked list */
shadow_defree_queue_t shadow_defrees;
@@ -608,4 +612,52 @@ H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2);
H5_DLL herr_t H5F__reparse_file_lock_variable_test(void);
#endif /* H5F_TESTING */
/* VFD SMWR LOG REPORTING MACROS */
/* H5F_POST_VFD_SWMR_LOG_ENTRY is the macro that needs to be used by the developers.
* It calls an internal reporting function H5F_post_vfd_swmr_log_entry() that receives
* the log entry_type_code, which generates the log tag, and the message log_info, which
* the library developer wants to save into the log file.
*
* The macro H5F_POST_VFD_SWMR_LOG_ENTRY_RELEASE(f, c, number_entry_production, m) is
* called by H5F_POST_VFD_SWMR_LOG_ENTRY when the HDF5 library is built with the
* production mode. Number_entry_production will control the number of entry tags that
* applications can receive. Currently this number is set to 1 and is subject to change
* when more tags are useful to be present to applications.
*
* The first argument of the macro is the HDF5 file pointer(H5F_t *).
* Its value needs to be checked to avoid a failure caused by "Low-Level File I/O "
* in the testhdf5 program, which involves the test of a non-existing HDF5 file.
*/
H5_DLL void H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info);
#define H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(fp, entry_type_code, log_info) \
do { \
if (fp != NULL) { \
if (fp->shared != NULL) { \
if (fp->shared->vfd_swmr_log_on == TRUE) { \
H5F_post_vfd_swmr_log_entry(fp, entry_type_code, log_info); \
} \
} \
} \
} while (0)
#define H5F_POST_VFD_SWMR_LOG_ENTRY_RELEASE(fp, entry_type_code, max_code, log_info) \
do { \
if (entry_type_code < max_code) { \
H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(fp, entry_type_code, log_info); \
} \
} while (0)
/* Note: change H5F_POST_VFD_SWMR_LOG_ENTRY_RELEASE(f, c, 1, m) on the following lines to
* H5F_POST_VFD_SWMR_LOG_ENTRY_RELEASE(f, c, your_number_entry_production, m)
* as necessary.
*/
#ifndef NDEBUG
#define H5F_POST_VFD_SWMR_LOG_ENTRY(f, c, m) H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(f, c, m)
#else
#define H5F_POST_VFD_SWMR_LOG_ENTRY(f, c, m) H5F_POST_VFD_SWMR_LOG_ENTRY_RELEASE(f, c, 1, m)
#endif
#endif /* H5Fpkg_H */

108
src/H5Fvfd_swmr.c
View File

@@ -54,6 +54,38 @@
#define nanosecs_per_second 1000000000 /* nanoseconds per second */
#define nanosecs_per_tenth_sec 100000000 /* nanoseconds per 0.1 second */
/* Declare an array of string to identify the VFD SMWR Log tags.
* Note this array is used to generate the entry tag by the log reporting macro
* H5F_POST_VFD_SWMR_LOG_ENTRY.
*
* The following is the first version. Developers can add/modify the tags as necessary.
*
* If the entry code is 0, H5Fvfd_swmr_log_tags[0] is used to report the entry tag.
* H5F_POST_VFD_SWMR_LOG_ENTRY(f, 0, log_msg) will put the log_msg attached to
* the entry tag "EOT_PROCESSING_TIME".
* The entry code number is listed in the comment for convenience.
* Currently for the production mode, only the "EOT_PROCESSING_TIME" is present.
*/
/* clang-format off */
static const char *H5Fvfd_swmr_log_tags[] = {
"EOT_PROCESSING_TIME", /* 0 */
"FILE_OPEN", /* 1 */
"FILE_CLOSE", /* 2 */
"EOT_TRIGGER_TIME", /* 3 */
"EOT_META_FILE_INDEX" /* 4 */
};
/* clang-format on */
/* This string defines the format of the VFD SWMR log file.
* The current maximum length of entry tag string is set to 26.
* One can enlarge or reduce this number as necessary.
* For example, to enlarge the maximum length of entry tag string to 30,
* Just change 26 to 30 in the following line, like
* const char *log_fmt_str="%-30s: %.3lf s: %s\n";
*/
const char *log_fmt_str = "%-26s: %.3lf s: %s\n";
/********************/
/* Local Prototypes */
/********************/
@@ -157,7 +189,6 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
/* Create the metadata file */
if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, O_CREAT | O_RDWR,
H5_POSIX_CREATE_MODE_RW))) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create the metadata file")
md_size = (hsize_t)shared->vfd_swmr_config.md_pages_reserved * shared->fs_page_size;
@@ -319,7 +350,16 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing)
if (H5F__vfd_swmr_update_end_of_tick_and_tick_num(shared, TRUE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick");
}
#if 1 /*Kent Save the end of close or flush info. to the log file, subject to comment out. */
H5F_POST_VFD_SWMR_LOG_ENTRY(f, 2, "VFD SWMR File close or flush ends");
#endif
done:
/* Kent: close the VFD SWMR log file if it is turned on */
if (shared->vfd_swmr_log_on) {
HDfclose(shared->vfd_swmr_log_file_ptr);
}
/* Kent */
FUNC_LEAVE_NOAPI(ret_value)
}
@@ -755,12 +795,24 @@ H5F_vfd_swmr_writer_end_of_tick(H5F_t *f, hbool_t wait_for_reader)
herr_t ret_value = SUCCEED; /* Return value */
hbool_t incr_tick = FALSE;
struct timespec start_time, end_time;
unsigned int temp_time;
char * log_msg;
FUNC_ENTER_NOAPI(FAIL)
HDassert(shared);
HDassert(shared->page_buf);
HDassert(shared->vfd_swmr_writer);
/* Kent */
/* Obtain the starting time for the logging info: the processing time of this function. */
if (shared->vfd_swmr_log_on == true) {
if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time via clock_gettime");
}
/* Kent */
if (!vfd_swmr_writer_may_increase_tick_to(shared->tick_num + 1, wait_for_reader))
goto update_eot;
@@ -883,6 +935,17 @@ update_eot:
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to insert entry into the EOT queue")
done:
/* Kent: Calcuate the processing time and write the time info to the log file */
if (shared->vfd_swmr_log_on == true) {
if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time via clock_gettime");
log_msg = HDmalloc(48);
temp_time = (unsigned int)(TOTAL_TIME_PASSED(start_time, end_time) * 1000);
HDsprintf(log_msg, "Writer time is %u milliseconds", temp_time);
H5F_POST_VFD_SWMR_LOG_ENTRY(f, 0, log_msg);
HDfree(log_msg);
}
/* Kent */
FUNC_LEAVE_NOAPI(ret_value)
}
@@ -1886,3 +1949,46 @@ H5F_vfd_swmr_process_eot_queue(hbool_t entering_api)
done:
FUNC_LEAVE_NOAPI(ret_value)
}
/*-------------------------------------------------------------------------
*
* Function: H5F_post_vfd_swmr_log_entry
*
* Purpose: Write the log information to the log file.
*
* Parameters:
* H5F_t *f IN: HDF5 file pointer
* int entry_type_code IN: The entry type code to identify the
* log entry tag.
* char *log_info IN: The information to be stored in the
* log file.
* Return: None
*
*-------------------------------------------------------------------------
*/
void
H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info)
{
double temp_time;
struct timespec current_time;
char * gettime_error;
/* Obtain the current time.
If failed, write an error message to the log file.
else calcluate the elapsed time in seconds since the log file
was created and wirte the time to the log file. */
if (HDclock_gettime(CLOCK_MONOTONIC, &current_time) < 0) {
gettime_error = HDmalloc(14);
HDsprintf(gettime_error, "gettime_error");
HDfprintf(f->shared->vfd_swmr_log_file_ptr, "%-26s: %s\n", H5Fvfd_swmr_log_tags[entry_type_code],
gettime_error);
HDfree(gettime_error);
}
else {
temp_time = TOTAL_TIME_PASSED(f->shared->vfd_swmr_log_start_time, current_time);
HDfprintf(f->shared->vfd_swmr_log_file_ptr, log_fmt_str, H5Fvfd_swmr_log_tags[entry_type_code],
temp_time, log_info);
}
return;
}

4
src/H5Pfapl.c
View File

@@ -5710,6 +5710,10 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long")
name_len = HDstrlen(config_ptr->log_file_path);
if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "log_file_path is too long")
/* Set the modified config */
if (H5P_set(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, config_ptr) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache initial config")

3
test/Makefile.am
View File

@@ -56,6 +56,7 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \
vfd_swmr_zoo_reader$(EXEEXT) vfd_swmr_zoo_writer$(EXEEXT) \
vfd_swmr_gperf_reader$(EXEEXT) vfd_swmr_gperf_writer$(EXEEXT) \
vfd_swmr_gfail_reader$(EXEEXT) vfd_swmr_gfail_writer$(EXEEXT) \
vfd_swmr_log_reader$(EXEEXT) vfd_swmr_log_writer$(EXEEXT) \
vds_env$(EXEEXT) \
vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT)
if HAVE_SHARED_CONDITIONAL
@@ -115,6 +116,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \
vfd_swmr_attrdset_reader vfd_swmr_attrdset_writer \
vfd_swmr_gperf_reader vfd_swmr_gperf_writer \
vfd_swmr_gfail_reader vfd_swmr_gfail_writer \
vfd_swmr_log_reader vfd_swmr_log_writer \
vfd_swmr_check_compat \
vfd_swmr_dsetchks_reader vfd_swmr_dsetchks_writer \
swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \
@@ -184,6 +186,7 @@ vfd_swmr_bigset_reader_SOURCES=vfd_swmr_bigset_writer.c
vfd_swmr_group_reader_SOURCES=vfd_swmr_group_writer.c
vfd_swmr_gperf_reader_SOURCES=vfd_swmr_gperf_writer.c
vfd_swmr_gfail_reader_SOURCES=vfd_swmr_gfail_writer.c
vfd_swmr_log_reader_SOURCES=vfd_swmr_log_writer.c
vfd_swmr_dsetops_reader_SOURCES=vfd_swmr_dsetops_writer.c
vfd_swmr_attrdset_writer_SOURCES=vfd_swmr_attrdset_writer.c

13
test/vfd_swmr_common.c
View File

@@ -369,6 +369,19 @@ init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t
} /* init_vfd_swmr_config() */
/* Initialize the log file path in config, this function should be called after init_vfd_swmr_config. */
void
init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...)
{
va_list ap;
HDva_start(ap, log_file_fmtstr);
evsnprintf(config->log_file_path, sizeof(config->log_file_path), log_file_fmtstr, ap);
HDva_end(ap);
} /* init_vfd_swmr_log() */
/* Perform common VFD SWMR configuration on the file-access property list:
* configure page buffering, set reasonable VFD SWMR defaults.
*/

3
test/vfd_swmr_common.h
View File

@@ -77,6 +77,9 @@ H5TEST_DLL void init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tic
hbool_t writer, hbool_t flush_raw_data, uint32_t md_pages_reserved,
const char *md_file_fmtstr, ...) H5_ATTR_FORMAT(printf, 7, 8);
H5TEST_DLL void init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...)
H5_ATTR_FORMAT(printf, 2, 3);
H5TEST_DLL hid_t vfd_swmr_create_fcpl(H5F_fspace_strategy_t fs_strategy, hsize_t fs_page_size);
H5TEST_DLL void dbgf(int, const char *, ...) H5_ATTR_FORMAT(printf, 2, 3);

2929
test/vfd_swmr_log_writer.c Normal file
View File

File diff suppressed because it is too large Load Diff