Merge branch 'develop' into cu_dev

2. temp fix open cu file driver only once

README.txt

@@ -16,6 +16,12 @@ project, tools and services can be found at the Group's website.
 
     https://www.hdfgroup.org/
 
+PURPOSE of cu_dev 
+------------------
+
+The motivation of cu_dev is to interface with Nvidia's GPUDirect Storage (GDS)
+API through a HDF5 VFD. You need to configure the hdf5 build with both 
+--with-cuda and --with-gds to enable the GDS VFD.
 
 DOCUMENTATION
 -------------

configure.ac

@@ -1363,6 +1363,43 @@ AC_ARG_WITH([fnord],
                          include/ and lib/ subdirectories
   ])
 
+####################### CUDA #######################
+AC_SUBST([ENABLE_CUDA])
+AC_SUBST([GDS_VFD])
+
+## Default is not present
+ENABLE_CUDA=no
+GDS_VFD=no
+
+m4_include([cuda.ac])
+
+if test -n "$CUDA_DIR"; then
+  CPPFLAGS="$CPPFLAGS -I$CUDA_DIR/include"
+  AM_CPPFLAGS="$AM_CPPFLAGS -I$CUDA_DIR/include"
+fi
+
+if test -n "$CUDA_LIBS"; then
+  LDFLAGS="$LDFLAGS -L$CUDA_DIR/lib64 $CUDA_LIBS"
+  AM_LDFLAGS="$AM_LDFLAGS -L$CUDA_DIR/lib64 $CUDA_LIBS"
+fi
+
+if test -n "$CUFILE_DIR"; then
+
+  if test x$ENABLE_CUDA = xyes; then
+    GDS_VFD=yes
+    AC_DEFINE([GDS_SUPPORT], [1], [Define if support for gds (cufile) is enabled])
+  else
+    GDS_VFD=no
+    AC_MSG_ERROR([GDS VFD was requested but missing cuda path, verify --with-cuda=])
+  fi
+
+  CPPFLAGS="$CPPFLAGS -I$CUFILE_DIR/lib"
+  AM_CPPFLAGS="$AM_CPPFLAGS -I$CUFILE_DIR/lib"
+  LDFLAGS="$LDFLAGS -L$CUFILE_DIR/lib -lcufile"
+  AM_LDFLAGS="$AM_LDFLAGS -L$CUFILE_DIR/lib -lcufile"
+fi
+##################### END CUDA #####################
+
 ## ----------------------------------------------------------------------
 ## Is dmalloc (debug malloc library) requested? It has a header file
 ## `dmalloc.h' and a library `-ldmalloc' and their locations might be

cuda.ac

@@ -0,0 +1,59 @@
+#
+########################################################
+
+# Allow setting CUDA compute capability at `configure`-time.
+AC_ARG_WITH([cuda-level],
+	[AS_HELP_STRING(
+		[--with-cuda-level=LEVEL],
+		[use CUDA compute capability LEVEL])],
+	[CUDA_LEVEL=$withval],
+	[CUDA_LEVEL=70])
+
+AC_ARG_WITH([cuda],
+	[AS_HELP_STRING([--with-cuda@<:@=PATH@:>@],
+		[use CUDA (installed in PATH) @<:@no@:>@])],
+	[CUDA_DIR=$withval],
+	[CUDA_DIR="no"])
+
+# We also support using the argument "auto" to mean "yes"
+case x$CUDA_DIR in
+             xno) ENABLE_CUDA=no ;;
+    x|xauto|xyes) ENABLE_CUDA=yes ;;
+               *) ENABLE_CUDA=yes ;;
+esac
+
+if test x$ENABLE_CUDA = xyes; then
+	AC_PATH_PROG([NVCC_PATH],nvcc,[],[$CUDA_DIR/bin])
+	if test -z "$NVCC_PATH" ; then
+		AC_MSG_ERROR(['bin/nvcc' not found in $CUDA_DIR/bin])
+	fi
+	
+	AC_CONFIG_FILES([cuda.mk])
+fi
+
+# Some CUDA installs only use the runtime library, so test for the presence of each
+if test x$ENABLE_CUDA = xyes; then
+	if test -e $CUDA_DIR/lib64/libcuda.so; then
+		CUDA_LIBS=-lcuda
+	else
+		if test -e $CUDA_DIR/lib64/libcudart.so; then
+			CUDA_LIBS=-lcudart
+		else
+			>&2 echo Could not find libcuda[rt].so
+			exit 1
+		fi
+	fi
+fi
+
+AC_ARG_WITH([cufile],
+    [AS_HELP_STRING([--with-cufile=DIR],
+    [Beta. Might change in the future.])],
+    [CUFILE_DIR=$withval],
+    [withval=no])
+
+AC_SUBST([ENABLE_CUDA])
+AC_SUBST([CUDA_LEVEL])
+AC_SUBST([CUDA_DIR])
+AC_SUBST([NVCC_PATH])
+AC_SUBST([CUDA_LIBS])
+AC_SUBST([CUFILE_DIR])

cuda.mk.in

@@ -0,0 +1,48 @@
+# Turn on if outer variable is set. Otherwise, allow user to set explicitly.
+CUDA_DEBUG   = $(DEBUG)
+CUDA_VERBOSE = $(VERBOSE)
+
+cuda_includes := -I@CUDA_DIR@/include
+#cuda_srcs     := $(srcdir)/src/H5FDsec2.c
+cuda_libs     := @CUDA_LIBS@
+cuda_ldflags  += -L@CUDA_DIR@/lib64
+cuda_objs     := $(patsubst %.cu,%.o,$(subst $(srcdir),$(build_dir),$(cuda_srcs)))
+
+# For more details on the architectures, see
+# http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#virtual-architecture-feature-list
+nvcc_flags += -arch=compute_@CUDA_LEVEL@ -code=sm_@CUDA_LEVEL@
+
+# nvcc includes are broken when the host compiler is gcc >= 5.0
+nvcc_flags += -D_FORCE_INLINES
+
+ifeq (1,$(CUDA_VERBOSE))
+	nvcc_flags += --ptxas-options=-v
+endif
+
+ifeq (1,$(CUDA_DEBUG))
+	nvcc_flags += --device-debug
+else
+	nvcc_flags += -use_fast_math
+endif
+
+.PHONY: show-config
+show-config: cuda-show-config
+
+.PHONY: cuda-show-config
+cuda-show-config:
+	@ echo NVCC_FLAGS = $(nvcc_flags)"\n"
+	@ echo CUDA_DEFINES = $(cuda_defines)"\n"
+	@ echo CUDA_INCLUDES = $(cuda_includes)"\n"
+	@ echo CUDA_LDFLAGS = $(cuda_ldflags)"\n"
+	@ echo CUDA_LIBS = $(cuda_libs)"\n"
+
+.PHONY: dist-clean
+dist-clean: cuda-dist-clean
+
+.PHONY: cuda-dist-clean
+cuda-dist-clean:
+	@ $(RM) cuda.mk
+
+$(build_dir)/%.o: $(srcdir)/src/%.cu
+	@ echo Compiling $<...
+	$(quiet) @NVCC_PATH@ -std=$(cxx_std) -Xcompiler "$(cxx_flags) $(depend_flags) $(depend_dir)/$*$(depend_suffix)" $(nvcc_flags) -c $< -o $@

src/H5FDgds.h

@@ -0,0 +1,63 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer:  John Ravi <jjravi@lbl.gov>
+ *              Wednesday, July  1, 2020
+ *
+ * Purpose:	The public header file for the CUDA GPUDirect Storage driver.
+ */
+#ifndef H5FDgds_H
+#define H5FDgds_H
+
+#define H5FD_GDS	(H5FD_gds_init())
+
+#define check_cudadrivercall(fn) \
+	do { \
+		CUresult res = fn; \
+		if (res != CUDA_SUCCESS) { \
+			const char *str = nullptr; \
+			cuGetErrorName(res, &str); \
+			fprintf(stderr, "cuda driver api call failed %d, %d : %s\n", fn, \
+				__LINE__, str); \
+			fprintf(stderr, "EXITING program!!!\n"); \
+			exit(1); \
+		} \
+	} while(0)
+
+#define check_cudaruntimecall(fn) \
+	do { \
+		cudaError_t res = fn; \
+		if (res != cudaSuccess) { \
+			const char *str = cudaGetErrorName(res); \
+			fprintf(stderr, "cuda runtime api call failed %d, %d : %s\n", fn, \
+				__LINE__, str); \
+			fprintf(stderr, "EXITING program!!!\n"); \
+			exit(1); \
+		} \
+	} while(0)
+
+/* Function prototypes */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+H5_DLL hid_t H5FD_gds_init(void);
+H5_DLL herr_t H5Pset_fapl_gds(hid_t fapl_id);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+

src/Makefile.am

@@ -61,8 +61,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \
         H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
         H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
         H5FAint.c H5FAstat.c H5FAtest.c \
-        H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \
+        H5FD.c H5FDcore.c H5FDfamily.c H5FDgds.c H5FDhdfs.c H5FDint.c \
-        H5FDmulti.c H5FDsec2.c H5FDspace.c \
+        H5FDlog.c H5FDmulti.c H5FDsec2.c H5FDspace.c \
         H5FDsplitter.c H5FDstdio.c H5FDtest.c \
         H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \
         H5FSstat.c H5FStest.c \
@@ -141,7 +141,7 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
         H5Apublic.h H5ACpublic.h \
         H5Cpublic.h H5Dpublic.h \
         H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \
-        H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \
+        H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDgds.h H5FDhdfs.h \
         H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \
         H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \
         H5Gpublic.h  H5Ipublic.h H5Lpublic.h \

src/hdf5.h

@@ -44,6 +44,7 @@
 #include "H5FDcore.h"           /* Files stored entirely in memory          */
 #include "H5FDdirect.h"         /* Linux direct I/O                         */
 #include "H5FDfamily.h"         /* File families                            */
+#include "H5FDgds.h"            /* CUDA GPUDirect Storage cufile I/O        */
 #include "H5FDhdfs.h"           /* Hadoop HDFS                              */
 #include "H5FDlog.h"            /* sec2 driver with I/O logging (for debugging) */
 #include "H5FDmirror.h"         /* Mirror VFD and IPC definitions           */

src/libhdf5.settings.in

@@ -64,6 +64,12 @@ Languages:
                            Java: @HDF_JAVA@
 @BUILD_JAVA_CONDITIONAL_TRUE@                  Java Compiler: @JAVA_VERSION@
 
+CUDA Options:
+-------------
+              Toolkit directory: @CUDA_DIR@
+                      nvcc path: @NVCC_PATH@
+                 Link libraries: @CUDA_LIBS@
+                  Compute level: @CUDA_LEVEL@
 
 Features:
 ---------
@@ -83,6 +89,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
                       Mirror VFD: @MIRROR_VFD@
               (Read-Only) S3 VFD: @ROS3_VFD@
             (Read-Only) HDFS VFD: @HAVE_LIBHDFS@
+                         GDS VFD: @GDS_VFD@
                          dmalloc: @HAVE_DMALLOC@
   Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@
                      API tracing: @TRACE_API@

test/h5test.c

@@ -963,6 +963,10 @@ h5_get_vfd_fapl(hid_t fapl)
         /* POSIX (section 2) read() and write() system calls */
         if(H5Pset_fapl_sec2(fapl) < 0)
             goto error;
+    } else if(!HDstrcmp(tok, "gds")) {
+        /* Standard C fread() and fwrite() system calls */
+        if(H5Pset_fapl_gds(fapl) < 0)
+            goto error;
     } else if(!HDstrcmp(tok, "stdio")) {
         /* Standard C fread() and fwrite() system calls */
         if(H5Pset_fapl_stdio(fapl) < 0)
@@ -1383,7 +1387,7 @@ h5_get_file_size(const char *filename, hid_t fapl)
             return(-1);
 
         /* Check for simple cases */
-        if(driver == H5FD_SEC2 || driver == H5FD_STDIO || driver == H5FD_CORE ||
+        if(driver == H5FD_SEC2 || driver == H5FD_GDS || driver == H5FD_STDIO || driver == H5FD_CORE ||
 #ifdef H5_HAVE_WINDOWS
                 driver == H5FD_WINDOWS ||
 #endif /* H5_HAVE_WINDOWS */