Fixed a bug in the links code where iterating over an empty group would

pass a NULL pointer to qsort(3), which is undefined behavior.

Fixes HDFFV-10829

release_docs/RELEASE.txt

@@ -375,6 +375,18 @@ Bug Fixes since HDF5-1.10.3 release
 
       (JTH - 2018/08/25, HDFFV-10501)
 
+    - When iterating over an old-style group (i.e., when not using the latest
+      file format) of size 0, a NULL pointer representing the empty links
+      table would be sent to qsort(3) for sorting, which is undefined behavior.
+
+      Iterating over an empty group is explicitly tested in the links test.
+      This has not caused any failures to date and was flagged by gcc's
+      -fsanitize=undefined.
+
+      The library no longer attempts to sort an empty array.
+
+      (DER - 2019/06/18, HDFFV-10829)
+
     Java Library:
     ----------------
     - JNI native library dependencies

src/H5Glink.c

@@ -402,15 +402,14 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5G__link_sort_table
+ * Function:    H5G__link_sort_table
  *
  * Purpose:     Sort table containing a list of links for a group
  *
- * Return:	Success:        Non-negative
- *		Failure:	Negative
+ * Return:      SUCCEED/FAIL
  *
- * Programmer:	Quincey Koziol
- *	        Nov 20, 2006
+ * Programmer:  Quincey Koziol
+ *              Nov 20, 2006
  *
  *-------------------------------------------------------------------------
  */
@@ -423,6 +422,13 @@ H5G__link_sort_table(H5G_link_table_t *ltable, H5_index_t idx_type,
     /* Sanity check */
     HDassert(ltable);
 
+    /* Can't sort when empty since the links table will be NULL */
+    if(0 == ltable->nlinks)
+        return SUCCEED;
+
+    /* This should never be NULL if the number of links is non-zero */
+    HDassert(ltable->lnks);
+
     /* Pick appropriate sorting routine */
     if(idx_type == H5_INDEX_NAME) {
         if(order == H5_ITER_INC)