philipp/hdf5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
81b1ed4e1724b8a6a731ab2b8fb03234a8b49d15
hdf5/src/H5FDs3comms.c
Jacob Smith 6a93d1f834 Fix improper S3 signing key null check (HDFFV-11015).
2020-02-07 13:47:15 -06:00

3594 lines
108 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*****************************************************************************
* Read-Only S3 Virtual File Driver (VFD)
*
* Source for S3 Communications module
*
* ***NOT A FILE DRIVER***
*
* Provide functions and structures required for interfacing with Amazon
* Simple Storage Service (S3).
*
* Provide S3 object access as if it were a local file.
*
* Connect to remote host, send and receive HTTP requests and responses
* as part of the AWS REST API, authenticating requests as appropriate.
*
* Programmer: Jacob Smith
* 2017-11-30
*
*****************************************************************************/
/****************/
/* Module Setup */
/****************/
/***********/
/* Headers */
/***********/
#include "H5private.h" /* generic functions */
#include "H5Eprivate.h" /* error handling */
#include "H5MMprivate.h" /* memory management */
#include "H5FDs3comms.h" /* S3 Communications */
/****************/
/* Local Macros */
/****************/
#ifdef H5_HAVE_ROS3_VFD
/* toggle debugging (enable with 1)
*/
#define S3COMMS_DEBUG 0
/* manipulate verbosity of CURL output
* operates separately from S3COMMS_DEBUG
*
* 0 -> no explicit curl output
* 1 -> on error, print failure info to stderr
* 2 -> in addition to above, print information for all performs; sets all
* curl handles with CURLOPT_VERBOSE
*/
#define S3COMMS_CURL_VERBOSITY 0
/* size to allocate for "bytes=<first_byte>[-<last_byte>]" HTTP Range value
*/
#define S3COMMS_MAX_RANGE_STRING_SIZE 128
/******************/
/* Local Typedefs */
/******************/
/********************/
/* Local Structures */
/********************/
/* struct s3r_datastruct
* Structure passed to curl write callback
* pointer to data region and record of bytes written (offset)
*/
struct s3r_datastruct {
unsigned long magic;
char *data;
size_t size;
};
#define S3COMMS_CALLBACK_DATASTRUCT_MAGIC 0x28c2b2ul
/********************/
/* Local Prototypes */
/********************/
size_t curlwritecallback(char *ptr,
size_t size,
size_t nmemb,
void *userdata);
herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
/*********************/
/* Package Variables */
/*********************/
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/*************/
/* Functions */
/*************/
/*----------------------------------------------------------------------------
*
* Function: curlwritecallback()
*
* Purpose:
*
* Function called by CURL to write received data.
*
* Writes bytes to `userdata`.
*
* Internally manages number of bytes processed.
*
* Return:
*
* - Number of bytes processed.
* - Should equal number of bytes passed to callback.
* - Failure will result in curl error: CURLE_WRITE_ERROR.
*
* Programmer: Jacob Smith
* 2017-08-17
*
*----------------------------------------------------------------------------
*/
size_t
curlwritecallback(char *ptr,
size_t size,
size_t nmemb,
void *userdata)
{
struct s3r_datastruct *sds = (struct s3r_datastruct *)userdata;
size_t product = (size * nmemb);
size_t written = 0;
if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) {
return written;
}
if (size > 0) {
HDmemcpy(&(sds->data[sds->size]), ptr, product);
sds->size += product;
written = product;
}
return written;
} /* end curlwritecallback() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_hrb_node_set()
*
* Purpose:
*
* Create, insert, modify, and remove elements in a field node list.
*
* `name` cannot be null; will return FAIL and list will be unaltered.
*
* Entries are accessed via the lowercase representation of their name:
* "Host", "host", and "hOSt" would all access the same node,
* but name's case is relevant in HTTP request output.
*
* List pointer `L` must always point to either of :
* - header node with lowest alphabetical order (by lowername)
* - NULL, if list is empty
*
* Types of operations:
*
* - CREATE
* - If `L` is NULL and `name` and `value` are not NULL,
* a new node is created at `L`, starting a list.
* - MODIFY
* - If a node is found with a matching lowercase name and `value`
* is not NULL, the existing name, value, and cat values are released
* and replaced with the new data.
* - No modifications are made to the list pointers.
* - REMOVE
* - If `value` is NULL, will attempt to remove node with matching
* lowercase name.
* - If no match found, returns FAIL and list is not modified.
* - When removing a node, all its resources is released.
* - If removing the last node in the list, list pointer is set to NULL.
* - INSERT
* - If no nodes exists with matching lowercase name and `value`
* is not NULL, a new node is created, inserted into list
* alphabetically by lowercase name.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - List was successfully modified
* - FAILURE: `FAIL`
* - Unable to perform operation
* - Forbidden (attempting to remove absent node, e.g.)
* - Internal error
*
* Programmer: Jacob Smith
* 2017-09-22
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_hrb_node_set(
hrb_node_t **L,
const char *name,
const char *value)
{
size_t i = 0;
char *valuecpy = NULL;
char *namecpy = NULL;
size_t namelen = 0;
char *lowername = NULL;
char *nvcat = NULL;
hrb_node_t *node_ptr = NULL;
hrb_node_t *new_node = NULL;
hbool_t is_looking = TRUE;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_hrb_node_set.\n");
HDprintf("NAME: %s\n", name);
HDprintf("VALUE: %s\n", value);
HDprintf("LIST:\n->");
for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) {
HDfprintf(stdout, "{%s}\n->", node_ptr->cat);
}
HDprintf("(null)\n");
fflush(stdout);
node_ptr = NULL;
#endif
if (name == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to operate on null name.\n");
}
namelen = HDstrlen(name);
/***********************
* PREPARE ALL STRINGS *
**********************/
/* copy and lowercase name
*/
lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
if (lowername == NULL) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"cannot make space for lowercase name copy.\n");
}
for (i = 0; i < namelen; i++) {
lowername[i] = (char)tolower((int)name[i]);
}
lowername[namelen] = 0;
/* If value supplied, copy name, value, and concatenated "name: value".
* If NULL, we will be removing a node or doing nothing, so no need for
* copies
*/
if (value != NULL) {
int ret = 0;
size_t valuelen = HDstrlen(value);
size_t catlen = namelen + valuelen + 2; /* +2 from ": " */
size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */
namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
if (namecpy == NULL) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"cannot make space for name copy.\n");
}
HDmemcpy(namecpy, name, (namelen + 1));
valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1));
if (valuecpy == NULL) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"cannot make space for value copy.\n");
}
HDmemcpy(valuecpy, value, (valuelen + 1));
nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite);
if (nvcat == NULL) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"cannot make space for concatenated string.\n");
}
ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value);
if (ret < 0 || (size_t)ret > catlen) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot concatenate `%s: %s", name, value);
}
HDassert( catlen == HDstrlen(nvcat) );
/* create new_node, should we need it
*/
new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t));
if (new_node == NULL) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"cannot make space for new set.\n");
}
new_node->magic = S3COMMS_HRB_NODE_MAGIC;
new_node->name = NULL;
new_node->value = NULL;
new_node->cat = NULL;
new_node->lowername = NULL;
new_node->next = NULL;
}
/***************
* ACT ON LIST *
***************/
if (*L == NULL) {
if (value == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"trying to remove node from empty list");
}
else {
#if S3COMMS_DEBUG
HDprintf("CREATE NEW\n"); fflush(stdout);
#endif
/*******************
* CREATE NEW LIST *
*******************/
new_node->cat = nvcat;
new_node->name = namecpy;
new_node->lowername = lowername;
new_node->value = valuecpy;
*L = new_node;
goto done; /* bypass further seeking */
}
}
/* sanity-check pointer passed in
*/
HDassert( (*L) != NULL );
HDassert( (*L)->magic == S3COMMS_HRB_NODE_MAGIC );
node_ptr = (*L);
/* Check whether to modify/remove first node in list
*/
if (strcmp(lowername, node_ptr->lowername) == 0) {
is_looking = FALSE;
if (value == NULL) {
#if S3COMMS_DEBUG
HDprintf("REMOVE HEAD\n"); fflush(stdout);
#endif
/***************
* REMOVE HEAD *
***************/
*L = node_ptr->next;
#if S3COMMS_DEBUG
HDprintf("FREEING CAT (node)\n"); fflush(stdout);
#endif
H5MM_xfree(node_ptr->cat);
#if S3COMMS_DEBUG
HDprintf("FREEING LOWERNAME (node)\n"); fflush(stdout);
#endif
H5MM_xfree(node_ptr->lowername);
#if S3COMMS_DEBUG
HDprintf("FREEING NAME (node)\n"); fflush(stdout);
#endif
H5MM_xfree(node_ptr->name);
#if S3COMMS_DEBUG
HDprintf("FREEING VALUE (node)\n"); fflush(stdout);
#endif
H5MM_xfree(node_ptr->value);
#if S3COMMS_DEBUG
HDprintf("MAGIC OK? %s\n",
(node_ptr->magic == S3COMMS_HRB_NODE_MAGIC) ? "YES" : "NO");
fflush(stdout);
#endif
HDassert( node_ptr->magic == S3COMMS_HRB_NODE_MAGIC );
node_ptr->magic += 1ul;
#if S3COMMS_DEBUG
HDprintf("FREEING POINTER\n"); fflush(stdout);
#endif
H5MM_xfree(node_ptr);
#if S3COMMS_DEBUG
HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout);
#endif
H5MM_xfree(lowername); lowername = NULL;
}
else {
#if S3COMMS_DEBUG
HDprintf("MODIFY HEAD\n"); fflush(stdout);
#endif
/***************
* MODIFY HEAD *
***************/
H5MM_xfree(node_ptr->cat);
H5MM_xfree(node_ptr->name);
H5MM_xfree(node_ptr->value);
node_ptr->name = namecpy;
node_ptr->value = valuecpy;
node_ptr->cat = nvcat;
H5MM_xfree(lowername);
lowername = NULL;
new_node->magic += 1ul;
H5MM_xfree(new_node);
new_node = NULL;
}
}
else
if (strcmp(lowername, node_ptr->lowername) < 0) {
is_looking = FALSE;
if (value == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"trying to remove a node 'before' head");
}
else {
#if S3COMMS_DEBUG
HDprintf("PREPEND NEW HEAD\n"); fflush(stdout);
#endif
/*******************
* INSERT NEW HEAD *
*******************/
new_node->name = namecpy;
new_node->value = valuecpy;
new_node->lowername = lowername;
new_node->cat = nvcat;
new_node->next = node_ptr;
*L = new_node;
}
}
/***************
* SEARCH LIST *
***************/
while (is_looking) {
if (node_ptr->next == NULL) {
is_looking = FALSE;
if (value == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"trying to remove absent node");
}
else {
#if S3COMMS_DEBUG
HDprintf("APPEND A NODE\n"); fflush(stdout);
#endif
/*******************
* APPEND NEW NODE *
*******************/
HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
new_node->name = namecpy;
new_node->value = valuecpy;
new_node->lowername = lowername;
new_node->cat = nvcat;
node_ptr->next = new_node;
}
}
else
if (strcmp(lowername, node_ptr->next->lowername) < 0) {
is_looking = FALSE;
if (value == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"trying to remove absent node");
}
else {
#if S3COMMS_DEBUG
HDprintf("INSERT A NODE\n"); fflush(stdout);
#endif
/*******************
* INSERT NEW NODE *
*******************/
HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
new_node->name = namecpy;
new_node->value = valuecpy;
new_node->lowername = lowername;
new_node->cat = nvcat;
new_node->next = node_ptr->next;
node_ptr->next = new_node;
}
}
else
if (strcmp(lowername, node_ptr->next->lowername) == 0) {
is_looking = FALSE;
if (value == NULL) {
/*****************
* REMOVE A NODE *
*****************/
hrb_node_t *tmp = node_ptr->next;
node_ptr->next = tmp->next;
#if S3COMMS_DEBUG
HDprintf("REMOVE A NODE\n"); fflush(stdout);
#endif
H5MM_xfree(tmp->cat);
H5MM_xfree(tmp->lowername);
H5MM_xfree(tmp->name);
H5MM_xfree(tmp->value);
HDassert( tmp->magic == S3COMMS_HRB_NODE_MAGIC );
tmp->magic += 1ul;
H5MM_xfree(tmp);
H5MM_xfree(lowername);
lowername = NULL;
}
else {
#if S3COMMS_DEBUG
HDprintf("MODIFY A NODE\n"); fflush(stdout);
#endif
/*****************
* MODIFY A NODE *
*****************/
node_ptr = node_ptr->next;
H5MM_xfree(node_ptr->name);
H5MM_xfree(node_ptr->value);
H5MM_xfree(node_ptr->cat);
HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
new_node->magic += 1ul;
H5MM_xfree(new_node);
H5MM_xfree(lowername);
new_node = NULL;
lowername = NULL;
node_ptr->name = namecpy;
node_ptr->value = valuecpy;
node_ptr->cat = nvcat;
}
}
else {
/****************
* KEEP LOOKING *
****************/
node_ptr = node_ptr->next;
}
} /* end while is_looking */
done:
if (ret_value == FAIL) {
/* clean up
*/
if (nvcat != NULL) { H5MM_xfree(nvcat); }
if (namecpy != NULL) { H5MM_xfree(namecpy); }
if (lowername != NULL) { H5MM_xfree(lowername); }
if (valuecpy != NULL) { H5MM_xfree(valuecpy); }
if (new_node != NULL) {
HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
new_node->magic += 1ul;
H5MM_xfree(new_node);
}
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_hrb_node_set() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_hrb_destroy()
*
* Purpose:
*
* Destroy and free resources _directly_ associated with an HTTP Buffer.
*
* Takes a pointer to pointer to the buffer structure.
* This allows for the pointer itself to be NULLed from within the call.
*
* If buffer or buffer pointer is NULL, there is no effect.
*
* Headers list at `first_header` is not touched.
*
* - Programmer should re-use or destroy `first_header` pointer
* (hrb_node_t *) as suits their purposes.
* - Recommend fetching prior to destroy()
* e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);`
* or maintaining an external reference.
* - Destroy node/list separately as appropriate
* - Failure to account for this will result in a memory leak.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - successfully released buffer resources
* - if `buf` is NULL or `*buf` is NULL, no effect
* - FAILURE: `FAIL`
* - `buf->magic != S3COMMS_HRB_MAGIC`
*
* Programmer: Jacob Smith
* 2017-07-21
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_hrb_destroy(hrb_t **_buf)
{
hrb_t *buf = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_hrb_destroy.\n");
#endif
if (_buf != NULL && *_buf != NULL) {
buf = *_buf;
if (buf->magic != S3COMMS_HRB_MAGIC) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"pointer's magic does not match.\n");
}
H5MM_xfree(buf->verb);
H5MM_xfree(buf->version);
H5MM_xfree(buf->resource);
buf->magic += 1ul;
H5MM_xfree(buf);
*_buf = NULL;
} /* end if `_buf` has some value */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_s3comms_hrb_destroy() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_hrb_init_request()
*
* Purpose:
*
* Create a new HTTP Request Buffer
*
* All non-null arguments should be null-terminated strings.
*
* If `verb` is NULL, defaults to "GET".
* If `http_version` is NULL, defaults to "HTTP/1.1".
*
* `resource` cannot be NULL; should be string beginning with slash
* character ('/').
*
* All strings are copied into the structure, making them safe from
* modification in source strings.
*
* Return:
*
* - SUCCESS: pointer to new `hrb_t`
* - FAILURE: `NULL`
*
* Programmer: Jacob Smith
* 2017-07-21
*
*----------------------------------------------------------------------------
*/
hrb_t *
H5FD_s3comms_hrb_init_request(const char *_verb,
const char *_resource,
const char *_http_version)
{
hrb_t *request = NULL;
char *res = NULL;
size_t reslen = 0;
hrb_t *ret_value = NULL;
char *verb = NULL;
size_t verblen = 0;
char *vrsn = NULL;
size_t vrsnlen = 0;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_hrb_init_request.\n");
#endif
if (_resource == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"resource string cannot be null.\n");
}
/* populate valid NULLs with defaults
*/
if (_verb == NULL) {
_verb = "GET";
}
if (_http_version == NULL) {
_http_version = "HTTP/1.1";
}
/* malloc space for and prepare structure
*/
request = (hrb_t *)H5MM_malloc(sizeof(hrb_t));
if (request == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
"no space for request structure");
}
request->magic = S3COMMS_HRB_MAGIC;
request->body = NULL;
request->body_len = 0;
request->first_header = NULL;
/* malloc and copy strings for the structure
*/
reslen = HDstrlen(_resource);
if (_resource[0] == '/') {
res = (char *)H5MM_malloc(sizeof(char) * (reslen+1));
if (res == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
"no space for resource string");
}
HDmemcpy(res, _resource, (reslen+1));
}
else {
res = (char *)H5MM_malloc(sizeof(char) * (reslen+2));
if (res == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
"no space for resource string");
}
*res = '/';
HDmemcpy((&res[1]), _resource, (reslen+1));
HDassert( (reslen+1) == HDstrlen(res) );
} /* end if (else resource string not starting with '/') */
verblen = HDstrlen(_verb) + 1;
verb = (char *)H5MM_malloc(sizeof(char) * verblen);
if (verb == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"no space for verb string");
}
HDstrncpy(verb, _verb, verblen);
vrsnlen = HDstrlen(_http_version) + 1;
vrsn = (char *)H5MM_malloc(sizeof(char) * vrsnlen);
if (vrsn == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"no space for http-version string");
}
HDstrncpy(vrsn, _http_version, vrsnlen);
/* place new copies into structure
*/
request->resource = res;
request->verb = verb;
request->version = vrsn;
ret_value = request;
done:
/* if there is an error, clean up after ourselves
*/
if (ret_value == NULL) {
if (request != NULL) H5MM_xfree(request);
if (vrsn != NULL) H5MM_xfree(vrsn);
if (verb != NULL) H5MM_xfree(verb);
if (res != NULL) H5MM_xfree(res);
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_s3comms_hrb_init_request() */
/****************************************************************************
* S3R FUNCTIONS
****************************************************************************/
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_s3r_close()
*
* Purpose:
*
* Close communications through given S3 Request Handle (`s3r_t`)
* and clean up associated resources.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* - fails if handle is null or has invalid magic number
*
*
* Programmer: Jacob Smith
* 2017-08-31
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_s3r_close(s3r_t *handle)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n");
#endif
if (handle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle cannot be null.\n");
}
if (handle->magic != S3COMMS_S3R_MAGIC) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has invalid magic.\n");
}
curl_easy_cleanup(handle->curlhandle);
H5MM_xfree(handle->secret_id);
H5MM_xfree(handle->region);
H5MM_xfree(handle->signing_key);
HDassert( handle->httpverb != NULL );
H5MM_xfree(handle->httpverb);
if (FAIL == H5FD_s3comms_free_purl(handle->purl)) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to release parsed url structure")
}
H5MM_xfree(handle);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD_s3comms_s3r_close */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_s3r_get_filesize()
*
* Purpose:
*
* Retrieve the filesize of an open request handle.
*
* Wrapper "getter" to hide implementation details.
*
*
* Return:
*
* - SUCCESS: size of file, in bytes, if handle is valid.
* - FAILURE: 0, if handle is NULL or undefined.
*
* Programmer: Jacob Smith 2017-01-14
*
*----------------------------------------------------------------------------
*/
size_t
H5FD_s3comms_s3r_get_filesize(s3r_t *handle)
{
size_t ret_value = 0;
FUNC_ENTER_NOAPI_NOINIT_NOERR
if (handle != NULL) {
ret_value = handle->filesize;
}
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD_s3comms_s3r_get_filesize */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_s3r_getsize()
*
* Purpose:
*
* Get the number of bytes of handle's target resource.
*
* Sets handle and curlhandle with to enact an HTTP HEAD request on file,
* and parses received headers to extract "Content-Length" from response
* headers, storing file size at `handle->filesize`.
*
* Critical step in opening (initiating) an `s3r_t` handle.
*
* Wraps `s3r_read()`.
* Sets curlhandle to write headers to a temporary buffer (using extant
* write callback) and provides no buffer for body.
*
* Upon exit, unsets HTTP HEAD settings from curl handle, returning to
* initial state. In event of error, curl handle state is undefined and is
* not to be trusted.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
*
* Programmer: Jacob Smith
* 2017-08-23
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_s3r_getsize(s3r_t *handle)
{
uintmax_t content_length = 0;
CURL *curlh = NULL;
char *end = NULL;
char *headerresponse = NULL;
struct s3r_datastruct sds = {
S3COMMS_CALLBACK_DATASTRUCT_MAGIC,
NULL,
0 };
char *start = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n");
#endif
if (handle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle cannot be null.\n");
}
if (handle->magic != S3COMMS_S3R_MAGIC) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has invalid magic.\n");
}
if (handle->curlhandle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has bad (null) curlhandle.\n")
}
/********************
* PREPARE FOR HEAD *
********************/
curlh = handle->curlhandle;
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_NOBODY,
1L) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"error while setting CURL option (CURLOPT_NOBODY). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_HEADERDATA,
&sds) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"error while setting CURL option (CURLOPT_HEADERDATA). "
"(placeholder flags)");
}
HDassert( handle->httpverb == NULL );
handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16);
if (handle->httpverb == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"unable to allocate space for S3 request HTTP verb");
}
HDmemcpy(handle->httpverb, "HEAD", 5);
headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER);
if (headerresponse == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"unable to allocate space for curl header response");
}
sds.data = headerresponse;
/*******************
* PERFORM REQUEST *
*******************/
/* these parameters fetch the entire file,
* but, with a NULL destination and NOBODY and HEADERDATA supplied above,
* only http metadata will be sent by server and recorded by s3comms
*/
if (FAIL ==
H5FD_s3comms_s3r_read(handle, 0, 0, NULL) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem in reading during getsize.\n");
}
if (sds.size > CURL_MAX_HTTP_HEADER) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"HTTP metadata buffer overrun\n");
} else if (sds.size == 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"No HTTP metadata\n");
#if S3COMMS_DEBUG
} else {
HDfprintf(stderr, "GETSIZE: OK\n");
#endif
}
/******************
* PARSE RESPONSE *
******************/
start = strstr(headerresponse,
"\r\nContent-Length: ");
if (start == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not find \"Content-Length\" in response.\n");
}
/* move "start" to beginning of value in line; find end of line
*/
start = start + HDstrlen("\r\nContent-Length: ");
end = strstr(start, "\r\n");
if (end == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not find end of content length line");
}
/* place null terminator at end of numbers
*/
*end = '\0';
content_length = strtoumax((const char *)start,
NULL,
0);
if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) {
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n");
}
if (content_length == 0 ||
errno == ERANGE) /* errno set by strtoumax*/
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not convert found \"Content-Length\" response (\"%s\")",
start); /* range is null-terminated, remember */
}
handle->filesize = (size_t)content_length;
/**********************
* UNDO HEAD SETTINGS *
**********************/
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_NOBODY,
NULL) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"error while setting CURL option (CURLOPT_NOBODY). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_HEADERDATA,
NULL) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"error while setting CURL option (CURLOPT_HEADERDATA). "
"(placeholder flags)");
}
done:
H5MM_xfree(headerresponse);
sds.magic += 1; /* set to bad magic */
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_s3r_getsize */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_s3r_open()
*
* Purpose:
*
* Logically 'open' a file hosted on S3.
*
* - create new Request Handle
* - copy supplied url
* - copy authentication info if supplied
* - create CURL handle
* - fetch size of file
* - connect with server and execute HEAD request
* - return request handle ready for reads
*
* To use 'default' port to connect, `port` should be 0.
*
* To prevent AWS4 authentication, pass null pointer to `region`, `id`,
* and `signing_key`.
*
* Uses `H5FD_s3comms_parse_url()` to validate and parse url input.
*
* Return:
*
* - SUCCESS: Pointer to new request handle.
* - FAILURE: NULL
* - occurs if:
* - authentication strings are inconsistent
* - must _all_ be null, or have at least `region` and `id`
* - url is NULL (no filename)
* - unable to parse url (malformed?)
* - error while performing `getsize()`
*
* Programmer: Jacob Smith
* 2017-09-01
*
*----------------------------------------------------------------------------
*/
s3r_t *
H5FD_s3comms_s3r_open(const char *url,
const char *region,
const char *id,
const unsigned char *signing_key)
{
size_t tmplen = 0;
CURL *curlh = NULL;
s3r_t *handle = NULL;
parsed_url_t *purl = NULL;
s3r_t *ret_value = NULL;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n");
#endif
if (url == NULL || url[0] == '\0') {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"url cannot be null.\n");
}
if (FAIL == H5FD_s3comms_parse_url(url, &purl)) {
/* probably a malformed url, but could be internal error */
HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL,
"unable to create parsed url structure");
}
HDassert( purl != NULL ); /* if above passes, this must be true */
HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t));
if (handle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
"could not malloc space for handle.\n");
}
handle->magic = S3COMMS_S3R_MAGIC;
handle->purl = purl;
handle->filesize = 0;
handle->region = NULL;
handle->secret_id = NULL;
handle->signing_key = NULL;
handle->httpverb = NULL;
/*************************************
* RECORD AUTHENTICATION INFORMATION *
*************************************/
if ((region != NULL && *region != '\0') ||
(id != NULL && *id != '\0') ||
(signing_key != NULL))
{
/* if one exists, all three must exist
*/
if (region == NULL || region[0] == '\0') {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"region cannot be null.\n");
}
if (id == NULL || id[0] == '\0') {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"secret id cannot be null.\n");
}
if (signing_key == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"signing key cannot be null.\n");
}
/* copy strings
*/
tmplen = HDstrlen(region) + 1;
handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen);
if (handle->region == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"could not malloc space for handle region copy.\n");
}
HDmemcpy(handle->region, region, tmplen);
tmplen = HDstrlen(id) + 1;
handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen);
if (handle->secret_id == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"could not malloc space for handle ID copy.\n");
}
HDmemcpy(handle->secret_id, id, tmplen);
tmplen = SHA256_DIGEST_LENGTH;
handle->signing_key =
(unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen);
if (handle->signing_key == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"could not malloc space for handle key copy.\n");
}
HDmemcpy(handle->signing_key, signing_key, tmplen);
} /* if authentication information provided */
/************************
* INITIATE CURL HANDLE *
************************/
curlh = curl_easy_init();
if (curlh == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"problem creating curl easy handle!\n");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_HTTPGET,
1L) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"error while setting CURL option (CURLOPT_HTTPGET). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_1_1) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"error while setting CURL option (CURLOPT_HTTP_VERSION). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_FAILONERROR,
1L) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"error while setting CURL option (CURLOPT_FAILONERROR). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_WRITEFUNCTION,
curlwritecallback) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"error while setting CURL option (CURLOPT_WRITEFUNCTION). "
"(placeholder flags)");
}
if ( CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_URL,
url) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"error while setting CURL option (CURLOPT_URL). "
"(placeholder flags)");
}
#if S3COMMS_CURL_VERBOSITY > 1
/* CURL will print (to stdout) information for each operation
*/
curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L);
#endif
handle->curlhandle = curlh;
/*******************
* OPEN CONNECTION *
* * * * * * * * * *
* GET FILE SIZE *
*******************/
if (FAIL ==
H5FD_s3comms_s3r_getsize(handle) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"problem in H5FD_s3comms_s3r_getsize.\n");
}
/*********************
* FINAL PREPARATION *
*********************/
HDassert( handle->httpverb != NULL );
HDmemcpy(handle->httpverb, "GET", 4);
ret_value = handle;
done:
if (ret_value == NULL) {
if (curlh != NULL) {
curl_easy_cleanup(curlh);
}
if (FAIL == H5FD_s3comms_free_purl(purl)) {
HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
"unable to free parsed url structure")
}
if (handle != NULL) {
H5MM_xfree(handle->region);
H5MM_xfree(handle->secret_id);
H5MM_xfree(handle->signing_key);
if (handle->httpverb != NULL) {
H5MM_xfree(handle->httpverb);
}
H5MM_xfree(handle);
}
}
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD_s3comms_s3r_open */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_s3r_read()
*
* Purpose:
*
* Read file pointed to by request handle, writing specified
* `offset` .. `offset + len` bytes to buffer `dest`.
*
* If `len` is 0, reads entirety of file starting at `offset`.
* If `offset` and `len` are both 0, reads entire file.
*
* If `offset` or `offset+len` is greater than the file size, read is
* aborted and returns `FAIL`.
*
* Uses configured "curl easy handle" to perform request.
*
* In event of error, buffer should remain unaltered.
*
* If handle is set to authorize a request, creates a new (temporary)
* HTTP Request object (hrb_t) for generating requisite headers,
* which is then translated to a `curl slist` and set in the curl handle
* for the request.
*
* `dest` _may_ be NULL, but no body data will be recorded.
*
* - In general practice, NULL should never be passed in as `dest`.
* - NULL `dest` passed in by internal function `s3r_getsize()`, in
* conjunction with CURLOPT_NOBODY to preempt transmission of file data
* from server.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
*
* Programmer: Jacob Smith
* 2017-08-22
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_s3r_read(s3r_t *handle,
haddr_t offset,
size_t len,
void *dest)
{
CURL *curlh = NULL;
CURLcode p_status = CURLE_OK;
struct curl_slist *curlheaders = NULL;
hrb_node_t *headers = NULL;
hrb_node_t *node = NULL;
struct tm *now = NULL;
char *rangebytesstr = NULL;
hrb_t *request = NULL;
int ret = 0; /* working variable to check */
/* return value of HDsnprintf */
struct s3r_datastruct *sds = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n");
#endif
/**************************************
* ABSOLUTELY NECESSARY SANITY-CHECKS *
**************************************/
if (handle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle cannot be null.\n");
}
if (handle->magic != S3COMMS_S3R_MAGIC) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has invalid magic.\n");
}
if (handle->curlhandle == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has bad (null) curlhandle.\n")
}
if (handle->purl == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle has bad (null) url.\n")
}
HDassert( handle->purl->magic == S3COMMS_PARSED_URL_MAGIC );
if (offset > handle->filesize || (len + offset) > handle->filesize) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to read past EoF")
}
curlh = handle->curlhandle;
/*********************
* PREPARE WRITEDATA *
*********************/
if (dest != NULL) {
sds = (struct s3r_datastruct *)H5MM_malloc(
sizeof(struct s3r_datastruct));
if (sds == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"could not malloc destination datastructure.\n");
}
sds->magic = S3COMMS_CALLBACK_DATASTRUCT_MAGIC;
sds->data = (char *)dest;
sds->size = 0;
if (CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_WRITEDATA,
sds) )
{
HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL,
"error while setting CURL option (CURLOPT_WRITEDATA). "
"(placeholder flags)");
}
}
/*********************
* FORMAT HTTP RANGE *
*********************/
if (len > 0) {
rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
(S3COMMS_MAX_RANGE_STRING_SIZE+1) );
if (rangebytesstr == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"could not malloc range format string.\n");
}
ret = HDsnprintf(rangebytesstr,
(S3COMMS_MAX_RANGE_STRING_SIZE),
"bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT,
offset,
offset + len - 1);
if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to format HTTP Range value");
} else if (offset > 0) {
rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
(S3COMMS_MAX_RANGE_STRING_SIZE+1));
if (rangebytesstr == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"could not malloc range format string.\n");
}
ret = HDsnprintf(rangebytesstr,
(S3COMMS_MAX_RANGE_STRING_SIZE),
"bytes="H5_PRINTF_HADDR_FMT"-",
offset);
if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to format HTTP Range value");
}
/*******************
* COMPILE REQUEST *
*******************/
if (handle->signing_key == NULL) {
/* Do not authenticate.
*/
if (rangebytesstr != NULL) {
/* Pass in range directly
*/
char *bytesrange_ptr = NULL; /* pointer past "bytes=" portion */
bytesrange_ptr = strchr(rangebytesstr, '=');
HDassert( bytesrange_ptr != NULL );
bytesrange_ptr++; /* move to first char past '=' */
HDassert( *bytesrange_ptr != '\0' );
if (CURLE_OK !=
curl_easy_setopt(curlh,
CURLOPT_RANGE,
bytesrange_ptr) )
{
HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL,
"error while setting CURL option (CURLOPT_RANGE). ");
}
}
} else {
/* authenticate request
*/
char authorization[512+1];
/* 512 := approximate max length...
* 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
* "SignedHeaders=,Signature=")>
* + 8 <yyyyMMDD>
* + 64 <hex(sha256())>
* + 128 <max? len(secret_id)>
* + 20 <max? len(region)>
* + 128 <max? len(signed_headers)>
*/
char buffer1[512+1]; /* -> Canonical Request -> Signature */
char buffer2[256+1]; /* -> String To Sign -> Credential */
char iso8601now[ISO8601_SIZE];
char signed_headers[48+1];
/* should be large enough for nominal listing:
* "host;range;x-amz-content-sha256;x-amz-date"
* + '\0', with "range;" possibly absent
*/
/* zero start of strings */
authorization[0] = 0;
buffer1[0] = 0;
buffer2[0] = 0;
iso8601now[0] = 0;
signed_headers[0] = 0;
/**** VERIFY INFORMATION EXISTS ****/
if (handle->region == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null region.\n");
}
if (handle->secret_id == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null secret_id.\n");
}
if (handle->signing_key == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null signing_key.\n");
}
if (handle->httpverb == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null httpverb.\n");
}
if (handle->purl->host == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null host.\n");
}
if (handle->purl->path == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"handle must have non-null resource.\n");
}
/**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/
request = H5FD_s3comms_hrb_init_request(
(const char *)handle->httpverb,
(const char *)handle->purl->path,
"HTTP/1.1");
if (request == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not allocate hrb_t request.\n");
}
HDassert( request->magic == S3COMMS_HRB_MAGIC );
now = gmnow();
if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not format ISO8601 time.\n");
}
if (FAIL ==
H5FD_s3comms_hrb_node_set(
&headers,
"x-amz-date",
(const char *)iso8601now) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to set x-amz-date header")
}
if (headers == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem building headers list. "
"(placeholder flags)\n");
}
HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
if (FAIL ==
H5FD_s3comms_hrb_node_set(
&headers,
"x-amz-content-sha256",
(const char *)EMPTY_SHA256) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to set x-amz-content-sha256 header")
}
if (headers == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem building headers list. "
"(placeholder flags)\n");
}
HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
if (rangebytesstr != NULL) {
if (FAIL ==
H5FD_s3comms_hrb_node_set(
&headers,
"Range",
(const char *)rangebytesstr) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to set range header")
}
if (headers == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem building headers list. "
"(placeholder flags)\n");
}
HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
}
if (FAIL ==
H5FD_s3comms_hrb_node_set(
&headers,
"Host",
(const char *)handle->purl->host) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to set host header")
}
if (headers == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem building headers list. "
"(placeholder flags)\n");
}
HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
request->first_header = headers;
/**** COMPUTE AUTHORIZATION ****/
if (FAIL == /* buffer1 -> canonical request */
H5FD_s3comms_aws_canonical_request(
buffer1,
512,
signed_headers,
48,
request) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"(placeholder flags)\n");
}
if ( FAIL == /* buffer2->string-to-sign */
H5FD_s3comms_tostringtosign(buffer2,
buffer1,
iso8601now,
handle->region) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"(placeholder flags)\n");
}
if (FAIL == /* buffer1 -> signature */
H5FD_s3comms_HMAC_SHA256(handle->signing_key,
SHA256_DIGEST_LENGTH,
buffer2,
HDstrlen(buffer2),
buffer1) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"(placeholder flags)\n");
}
iso8601now[8] = 0; /* trim to yyyyMMDD */
ret = S3COMMS_FORMAT_CREDENTIAL(buffer2,
handle->secret_id,
iso8601now,
handle->region,
"s3");
if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to format aws4 credential string");
ret = HDsnprintf(
authorization,
512,
"AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
buffer2,
signed_headers,
buffer1);
if (ret <= 0 || ret >= 512) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to format aws4 authorization string");
}
/* append authorization header to http request buffer
*/
if (H5FD_s3comms_hrb_node_set(
&headers,
"Authorization",
(const char *)authorization)
== FAIL)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to set Authorization header")
}
if (headers == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem building headers list. "
"(placeholder flags)\n");
}
/* update hrb's "first header" pointer
*/
request->first_header = headers;
/**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/
node = request->first_header;
while (node != NULL) {
HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC );
curlheaders = curl_slist_append(curlheaders,
(const char *)node->cat);
if (curlheaders == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not append header to curl slist. "
"(placeholder flags)\n");
}
node = node->next;
}
/* sanity-check
*/
if (curlheaders == NULL) {
/* above loop was probably never run */
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"curlheaders was never populated.\n");
}
/* finally, set http headers in curl handle
*/
if (curl_easy_setopt(
curlh,
CURLOPT_HTTPHEADER,
curlheaders)
!= CURLE_OK)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"error while setting CURL option "
"(CURLOPT_HTTPHEADER). (placeholder flags)");
}
} /* end if should authenticate (info provided) */
/*******************
* PERFORM REQUEST *
*******************/
#if S3COMMS_CURL_VERBOSITY > 0
/* In event of error, print detailed information to stderr
* This is not the default behavior.
*/
{
long int httpcode = 0;
char curlerrbuf[CURL_ERROR_SIZE];
curlerrbuf[0] = '\0';
if (CURLE_OK !=
curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem setting error buffer")
}
p_status = curl_easy_perform(curlh);
if (p_status != CURLE_OK) {
if (CURLE_OK !=
curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem getting response code")
}
HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n",
p_status, httpcode);
HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status));
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
"problem while performing request.\n");
}
if (CURLE_OK !=
curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL) )
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem unsetting error buffer")
}
} /* verbose error reporting */
#else
p_status = curl_easy_perform(curlh);
if (p_status != CURLE_OK) {
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
"curl cannot perform request\n")
}
#endif
#if S3COMMS_DEBUG
if (dest != NULL) {
HDfprintf(stderr, "len: %d\n", (int)len);
HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n");
if (sds == NULL) {
HDfprintf(stderr, "sds is NULL!\n");
}
else {
HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds);
HDfprintf(stderr, "sds->size: %d\n", (int)sds->size);
if (len > sds->size) {
HDfprintf(stderr, "buffer overwrite\n");
}
}
}
else {
HDfprintf(stderr, "performed on entire file\n");
}
#endif
done:
/* clean any malloc'd resources
*/
if (curlheaders != NULL) {
curl_slist_free_all(curlheaders);
curlheaders = NULL;
}
if (rangebytesstr != NULL) {
H5MM_xfree(rangebytesstr);
rangebytesstr = NULL;
}
if (sds != NULL) {
H5MM_xfree(sds);
sds = NULL;
}
if (request != NULL) {
while (headers != NULL)
if (FAIL ==
H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL))
{
HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot release header node")
}
HDassert( NULL == headers );
if (FAIL == H5FD_s3comms_hrb_destroy(&request)) {
HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot release header request structure")
}
HDassert( NULL == request );
}
if (curlh != NULL) {
/* clear any Range */
if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) {
HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot unset CURLOPT_RANGE")
}
/* clear headers */
if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) {
HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot unset CURLOPT_HTTPHEADER")
}
}
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_s3r_read */
/****************************************************************************
* MISCELLANEOUS FUNCTIONS
****************************************************************************/
/*----------------------------------------------------------------------------
*
* Function: gmnow()
*
* Purpose:
*
* Get the output of `time.h`'s `gmtime()` call while minimizing setup
* clutter where important.
*
* Return:
*
* Pointer to resulting `struct tm`,as created by gmtime(time_t * T).
*
* Programmer: Jacob Smith
* 2017-07-12
*
*----------------------------------------------------------------------------
*/
struct tm *
gmnow(void)
{
time_t now;
time_t *now_ptr = &now;
struct tm *ret_value = NULL;
/* Doctor assert, checks against error in time() */
if ( (time_t)(-1) != time(now_ptr) ) {
ret_value = gmtime(now_ptr);
}
HDassert( ret_value != NULL );
return ret_value;
} /* end gmnow() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_aws_canonical_request()
*
* Purpose:
*
* Compose AWS "Canonical Request" (and signed headers string)
* as defined in the REST API documentation.
*
* Both destination strings are null-terminated.
*
* Destination string arguments must be provided with adequate space.
*
* Canonical Request format:
*
* <HTTP VERB>"\n"
* <resource path>"\n"
* <query string>"\n"
* <header1>"\n" (`lowercase(name)`":"`trim(value)`)
* <header2>"\n"
* ... (headers sorted by name)
* <header_n>"\n"
* "\n"
* <signed headers>"\n" (`lowercase(header 1 name)`";"`header 2 name`;...)
* <hex-string of sha256sum of body> ("e3b0c4429...", e.g.)
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - writes canonical request to respective `...dest` strings
* - FAILURE: `FAIL`
* - one or more input argument was NULL
* - internal error
*
* Programmer: Jacob Smith
* 2017-10-04
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_aws_canonical_request(
char *canonical_request_dest,
int _cr_size,
char *signed_headers_dest,
int _sh_size,
hrb_t *http_request)
{
hrb_node_t *node = NULL;
const char *query_params = ""; /* unused at present */
herr_t ret_value = SUCCEED;
int ret = 0;
size_t cr_size = (size_t)_cr_size;
size_t sh_size = (size_t)_sh_size;
size_t cr_len = 0; /* working length of canonical request str */
size_t sh_len = 0; /* working length of signed headers str */
char tmpstr[256+1];
tmpstr[256] = 0; /* terminating NULL */
/* "query params" refers to the optional element in the URL, e.g.
* http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
* ^-----------------^
*
* Not handled/implemented as of 2017-10-xx.
* Element introduced as empty placeholder and reminder.
* Further research to be done if this is ever relevant for the
* VFD use-cases.
*/
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_aws_canonical_request.\n");
#endif
if (http_request == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"hrb object cannot be null.\n");
}
HDassert( http_request->magic == S3COMMS_HRB_MAGIC );
if (canonical_request_dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"canonical request destination cannot be null.\n");
}
if (signed_headers_dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"signed headers destination cannot be null.\n");
}
/* HTTP verb, resource path, and query string lines
*/
cr_len = (HDstrlen(http_request->verb) +
HDstrlen(http_request->resource) +
HDstrlen(query_params) +
(size_t)3); /* three newline chars */
if (cr_len >= cr_size) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"not enough space in canonical request");
}
ret = HDsnprintf( /* TODO: compiler warning */
canonical_request_dest,
(cr_size-1),
"%s\n%s\n%s\n",
http_request->verb,
http_request->resource,
query_params);
if (ret < 0 || (size_t)ret >= cr_size) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to compose canonical request first line");
}
/* write in canonical headers, building signed headers concurrently
*/
node = http_request->first_header; /* assumed sorted */
while (node != NULL) {
HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC);
ret = HDsnprintf(
tmpstr,
256,
"%s:%s\n",
node->lowername,
node->value);
if (ret < 0 || ret >= 256) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to concatenate HTTP header %s:%s",
node->lowername,
node->value);
}
cr_len += HDstrlen(tmpstr);
if (cr_len + 1 > cr_size) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"not enough space in canonical request");
}
strcat(canonical_request_dest, tmpstr);
ret = HDsnprintf(
tmpstr,
256,
"%s;",
node->lowername);
if (ret < 0 || ret >= 256) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to append semicolon to lowername %s",
node->lowername);
}
sh_len += HDstrlen(tmpstr);
if (sh_len + 1 > sh_size) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"not enough space in signed headers");
}
strcat(signed_headers_dest, tmpstr);
node = node->next;
} /* end while node is not NULL */
/* remove tailing ';' from signed headers sequence
*/
signed_headers_dest[HDstrlen(signed_headers_dest) - 1] = '\0';
/* append signed headers and payload hash
* NOTE: at present, no HTTP body is handled, per the nature of
* requests/range-gets
*/
strcat(canonical_request_dest, "\n");
strcat(canonical_request_dest, signed_headers_dest);
strcat(canonical_request_dest, "\n");
strcat(canonical_request_dest, EMPTY_SHA256);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_aws_canonical_request() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_bytes_to_hex()
*
* Purpose:
*
* Produce human-readable hex string [0-9A-F] from sequence of bytes.
*
* For each byte (char), writes two-character hexadecimal representation.
*
* No null-terminator appended.
*
* Assumes `dest` is allocated to enough size (msg_len * 2).
*
* Fails if either `dest` or `msg` are null.
*
* `msg_len` message length of 0 has no effect.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - hex string written to `dest` (not null-terminated)
* - FAILURE: `FAIL`
* - `dest == NULL`
* - `msg == NULL`
*
* Programmer: Jacob Smith
* 2017-07-12
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_bytes_to_hex(
char *dest,
const unsigned char *msg,
size_t msg_len,
hbool_t lowercase)
{
size_t i = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_bytes_to_hex.\n");
#endif
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"hex destination cannot be null.\n")
}
if (msg == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"bytes sequence cannot be null.\n")
}
for (i = 0; i < msg_len; i++) {
int chars_written = HDsnprintf(&(dest[i * 2]),
3, /* 'X', 'X', '\n' */
(lowercase == TRUE) ? "%02x" : "%02X",
msg[i]);
if (chars_written != 2) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem while writing hex chars for %c",
msg[i]);
}
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_bytes_to_hex() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_free_purl()
*
* Purpose:
*
* Release resources from a parsed_url_t pointer.
*
* If pointer is null, nothing happens.
*
* Return:
*
* `SUCCEED` (never fails)
*
* Programmer: Jacob Smith
* 2017-11-01
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_free_purl(parsed_url_t *purl)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
#if S3COMMS_DEBUG
HDprintf("called H5FD_s3comms_free_purl.\n");
#endif
if (purl != NULL) {
HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); }
if (purl->host != NULL) { H5MM_xfree(purl->host); }
if (purl->port != NULL) { H5MM_xfree(purl->port); }
if (purl->path != NULL) { H5MM_xfree(purl->path); }
if (purl->query != NULL) { H5MM_xfree(purl->query); }
purl->magic += 1ul;
H5MM_xfree(purl);
}
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_s3comms_free_purl() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_HMAC_SHA256()
*
* Purpose:
*
* Generate Hash-based Message Authentication Checksum using the SHA-256
* hashing algorithm.
*
* Given a key, message, and respective lengths (to accommodate null
* characters in either), generate _hex string_ of authentication checksum
* and write to `dest`.
*
* `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size.
* Not enforceable by this function.
* `dest` will _not_ be null-terminated by this function.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - hex string written to `dest` (not null-terminated)
* - FAILURE: `FAIL`
* - `dest == NULL`
* - error while generating hex string output
*
* Programmer: Jacob Smith
* 2017-07-??
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_HMAC_SHA256(
const unsigned char *key,
size_t key_len,
const char *msg,
size_t msg_len,
char *dest)
{
unsigned char md[SHA256_DIGEST_LENGTH];
unsigned int md_len = SHA256_DIGEST_LENGTH;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n");
#endif
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"destination cannot be null.");
}
HMAC(EVP_sha256(),
key,
(int)key_len,
(const unsigned char *)msg,
msg_len,
md,
&md_len);
if (H5FD_s3comms_bytes_to_hex(
dest,
(const unsigned char *)md,
(size_t)md_len,
true)
== FAIL)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not convert to hex string.");
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_HMAC_SHA256 */
/*-----------------------------------------------------------------------------
*
* Function: H5FD__s3comms_load_aws_creds_from_file()
*
* Purpose:
*
* Extract AWS configuration information from a target file.
*
* Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate
* that region in the file. If not found, returns in error and output
* pointers are not modified.
*
* If the profile label is found, attempts to locate and parse configuration
* data, stopping at the first line where:
* + reached end of file
* + line does not start with a recognized setting name
*
* Following AWS documentation, looks for any of:
* + aws_access_key_id
* + aws_secret_access_key
* + region
*
* To be valid, the setting must begin the line with one of the keywords,
* followed immediately by an equals sign '=', and have some data before
* newline at end of line.
* + `spam=eggs` would be INVALID because name is unrecognized
* + `region = us-east-2` would be INVALID because of spaces
* + `region=` would be INVALID because no data.
*
* Upon successful parsing of a setting line, will store the result in the
* corresponding output pointer. If the output pointer is NULL, will skip
* any matching setting line while parsing -- useful to prevent overwrite
* when reading from multiple files.
*
* Return:
*
* + SUCCESS: `SUCCEED`
* + no error. settings may or may not have been loaded.
* + FAILURE: `FAIL`
* + internal error occurred.
* + -1 :: unable to format profile label
* + -2 :: profile name/label not found in file
* + -3 :: some other error
*
* Programmer: Jacob Smith
* 2018-02-27
*
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__s3comms_load_aws_creds_from_file(
FILE *file,
const char *profile_name,
char *key_id,
char *access_key,
char *aws_region)
{
char profile_line[32];
char buffer[128];
const char *setting_names[] = {
"region",
"aws_access_key_id",
"aws_secret_access_key",
};
char * const setting_pointers[] = {
aws_region,
key_id,
access_key,
};
unsigned setting_count = 3;
herr_t ret_value = SUCCEED;
unsigned buffer_i = 0;
unsigned setting_i = 0;
int found_setting = 0;
char *line_buffer = &(buffer[0]);
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called load_aws_creds_from_file.\n");
#endif
/* format target line for start of profile */
if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to format profile label")
}
/* look for start of profile */
do {
/* clear buffer */
for (buffer_i=0; buffer_i < 128; buffer_i++) {
buffer[buffer_i] = 0;
}
line_buffer = fgets(line_buffer, 128, file);
if (line_buffer == NULL) { /* reached end of file */
goto done;
}
} while (strncmp(line_buffer, profile_line, HDstrlen(profile_line)));
/* extract credentials from lines */
do {
/* clear buffer */
for (buffer_i=0; buffer_i < 128; buffer_i++) {
buffer[buffer_i] = 0;
}
/* collect a line from file */
line_buffer = fgets(line_buffer, 128, file);
if (line_buffer == NULL) {
goto done; /* end of file */
}
/* loop over names to see if line looks like assignment */
for (setting_i = 0; setting_i < setting_count; setting_i++) {
size_t setting_name_len = 0;
const char *setting_name = NULL;
char line_prefix[128];
setting_name = setting_names[setting_i];
setting_name_len = HDstrlen(setting_name);
if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to format line prefix")
}
/* found a matching name? */
if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) {
found_setting = 1;
/* skip NULL destination buffer */
if (setting_pointers[setting_i] == NULL) {
break;
}
/* advance to end of name in string */
do {
line_buffer++;
} while (*line_buffer != 0 && *line_buffer != '=');
if (*line_buffer == 0 || *(line_buffer+1) == 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"incomplete assignment in file")
}
line_buffer++; /* was pointing at '='; advance */
/* copy line buffer into out pointer */
if (HDstrncpy(
setting_pointers[setting_i],
(const char *)line_buffer,
HDstrlen(line_buffer))
== NULL)
{
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to copy line into pointer")
}
/* "trim" tailing whitespace by replacing with null terminator*/
buffer_i = 0;
while (!isspace(setting_pointers[setting_i][buffer_i])) {
buffer_i++;
}
setting_pointers[setting_i][buffer_i] = '\0';
break; /* have read setting; don't compare with others */
} /* end if possible name match */
} /* end for each setting name */
} while (found_setting);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__s3comms_load_aws_creds_from_file() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_load_aws_profile()
*
* Purpose :
*
* Read aws profile elements from standard location on system and store
* settings in memory.
*
* Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard
* files for AWS tools. If a file exists (can be opened), looks for the
* given profile name and reads the settings into the relevant buffer.
*
* Any setting duplicated in both files will be set to that from
* `credentials`.
*
* Settings are stored in the supplied buffers as null-terminated strings.
*
* Return:
*
* + SUCCESS: `SUCCEED` (0)
* + no error occurred and all settings were populated
* + FAILURE: `FAIL` (-1)
* + internal error occurred
* + unable to locate profile
* + region, key id, and secret key were not all found and set
*
* Programmer: Jacob Smith
* 2018-02-27
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_load_aws_profile(const char *profile_name,
char *key_id_out,
char *secret_access_key_out,
char *aws_region_out)
{
herr_t ret_value = SUCCEED;
FILE *credfile = NULL;
char awspath[117];
char filepath[128];
int ret = 0;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_load_aws_profile.\n");
#endif
#ifdef H5_HAVE_WIN32_API
ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")) ;
#else
ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ;
#endif
if (ret < 0 || (size_t)ret >= 117) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to format home-aws path")
}
ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials");
if (ret < 0 || (size_t)ret >= 128) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to format credentials path")
}
credfile = fopen(filepath, "r");
if (credfile != NULL) {
if (H5FD__s3comms_load_aws_creds_from_file(
credfile,
profile_name,
key_id_out,
secret_access_key_out,
aws_region_out)
== FAIL)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to load from aws credentials")
}
if (fclose(credfile) == EOF) {
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
"unable to close credentials file")
}
credfile = NULL;
} /* end if credential file opened */
ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config");
if (ret < 0 || (size_t)ret >= 128) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
"unable to format config path")
}
credfile = fopen(filepath, "r");
if (credfile != NULL) {
if (H5FD__s3comms_load_aws_creds_from_file(
credfile,
profile_name,
(*key_id_out == 0) ? key_id_out : NULL,
(*secret_access_key_out == 0) ? secret_access_key_out : NULL,
(*aws_region_out == 0) ? aws_region_out : NULL)
== FAIL)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to load from aws config")
}
if (fclose(credfile) == EOF) {
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
"unable to close config file")
}
credfile = NULL;
} /* end if credential file opened */
/* fail if not all three settings were loaded */
if (*key_id_out == 0 ||
*secret_access_key_out == 0 ||
*aws_region_out == 0)
{
ret_value = FAIL;
}
done:
if (credfile != NULL) {
if (fclose(credfile) == EOF) {
HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL,
"problem error-closing aws configuration file")
}
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_load_aws_profile() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_nlowercase()
*
* Purpose:
*
* From string starting at `s`, write `len` characters to `dest`,
* converting all to lowercase.
*
* Behavior is undefined if `s` is NULL or `len` overruns the allocated
* space of either `s` or `dest`.
*
* Provided as convenience.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - upon completion, `dest` is populated
* - FAILURE: `FAIL`
* - `dest == NULL`
*
* Programmer: Jacob Smith
* 2017-09-18
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_nlowercase(
char *dest,
const char *s,
size_t len)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_nlowercase.\n");
#endif
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"destination cannot be null.\n");
}
if (len > 0) {
HDmemcpy(dest, s, len);
do {
len--;
dest[len] = (char)tolower( (int)dest[len] );
} while (len > 0);
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_nlowercase() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_parse_url()
*
* Purpose:
*
* Parse URL-like string and stuff URL components into
* `parsed_url` structure, if possible.
*
* Expects null-terminated string of format:
* SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY]
* where SCHEME :: "[a-zA-Z/.-]+"
* PORT :: "[0-9]"
*
* Stores resulting structure in argument pointer `purl`, if successful,
* creating and populating new `parsed_url_t` structure pointer.
* Empty or absent elements are NULL in new purl structure.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - `purl` pointer is populated
* - FAILURE: `FAIL`
* - unable to parse
* - `purl` is unaltered (probably NULL)
*
* Programmer: Jacob Smith
* 2017-10-30
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_parse_url(
const char *str,
parsed_url_t **_purl)
{
parsed_url_t *purl = NULL; /* pointer to new structure */
const char *tmpstr = NULL; /* working pointer in string */
const char *curstr = str; /* "start" pointer in string */
long int len = 0; /* substring length */
long int urllen = 0; /* length of passed-in url string */
unsigned int i = 0;
herr_t ret_value = FAIL;
FUNC_ENTER_NOAPI_NOINIT;
#if S3COMMS_DEBUG
HDprintf("called H5FD_s3comms_parse_url.\n");
#endif
if (str == NULL || *str == '\0') {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"invalid url string");
}
urllen = (long int)HDstrlen(str);
purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t));
if (purl == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for parsed_url_t");
}
purl->magic = S3COMMS_PARSED_URL_MAGIC;
purl->scheme = NULL;
purl->host = NULL;
purl->port = NULL;
purl->path = NULL;
purl->query = NULL;
/***************
* READ SCHEME *
***************/
tmpstr = HDstrchr(curstr, ':');
if (tmpstr == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"invalid SCHEME construction: probably not URL");
}
len = tmpstr - curstr;
HDassert( (0 <= len) && (len < urllen) );
/* check for restrictions
*/
for (i = 0; i < len; i++) {
/* scheme = [a-zA-Z+-.]+ (terminated by ":") */
if (!isalpha(curstr[i]) &&
'+' != curstr[i] &&
'-' != curstr[i] &&
'.' != curstr[i])
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"invalid SCHEME construction");
}
}
/* copy lowercased scheme to structure
*/
purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
if (purl->scheme == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for SCHEME");
}
(void)HDstrncpy(purl->scheme, curstr, (size_t)len);
purl->scheme[len] = '\0';
for ( i = 0; i < len; i++ ) {
purl->scheme[i] = (char)tolower(purl->scheme[i]);
}
/* Skip "://" */
tmpstr += 3;
curstr = tmpstr;
/*************
* READ HOST *
*************/
if (*curstr == '[') {
/* IPv6 */
while (']' != *tmpstr) {
if (tmpstr == 0) { /* end of string reached! */
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"reached end of URL: incomplete IPv6 HOST");
}
tmpstr++;
}
tmpstr++;
} /* end if (IPv6) */
else {
while (0 != *tmpstr) {
if (':' == *tmpstr ||
'/' == *tmpstr ||
'?' == *tmpstr)
{
break;
}
tmpstr++;
}
} /* end else (IPv4) */
len = tmpstr - curstr;
if (len == 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"HOST substring cannot be empty");
}
else
if (len > urllen) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem with length of HOST substring");
}
/* copy host
*/
purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
if (purl->host == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for HOST");
}
(void)HDstrncpy(purl->host, curstr, (size_t)len);
purl->host[len] = 0;
/*************
* READ PORT *
*************/
if (':' == *tmpstr) {
tmpstr += 1; /* advance past ':' */
curstr = tmpstr;
while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr)) {
tmpstr++;
}
len = tmpstr - curstr;
if (len == 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"PORT element cannot be empty");
}
else
if (len > urllen) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem with length of PORT substring");
}
for (i = 0; i < len; i ++) {
if (!isdigit(curstr[i])) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"PORT is not a decimal string");
}
}
/* copy port
*/
purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
if (purl->port == NULL) { /* cannot malloc */
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for PORT");
}
(void)HDstrncpy(purl->port, curstr, (size_t)len);
purl->port[len] = 0;
} /* end if PORT element */
/*************
* READ PATH *
*************/
if ('/' == *tmpstr) {
/* advance past '/' */
tmpstr += 1;
curstr = tmpstr;
/* seek end of PATH
*/
while ((0 != *tmpstr) && ('?' != *tmpstr)) {
tmpstr++;
}
len = tmpstr - curstr;
if (len > urllen) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem with length of PATH substring");
}
if (len > 0) {
purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
if (purl->path == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for PATH");
} /* cannot malloc path pointer */
(void)HDstrncpy(purl->path, curstr, (size_t)len);
purl->path[len] = 0;
}
} /* end if PATH element */
/**************
* READ QUERY *
**************/
if ('?' == *tmpstr) {
tmpstr += 1;
curstr = tmpstr;
while (0 != *tmpstr) {
tmpstr++;
}
len = tmpstr - curstr;
if (len == 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"QUERY cannot be empty");
} else if (len > urllen) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem with length of QUERY substring");
}
purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
if (purl->query == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
"can't allocate space for QUERY");
} /* cannot malloc path pointer */
(void)HDstrncpy(purl->query, curstr, (size_t)len);
purl->query[len] = 0;
} /* end if QUERY exists */
*_purl = purl;
ret_value = SUCCEED;
done:
if (ret_value == FAIL) {
H5FD_s3comms_free_purl(purl);
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_parse_url() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_percent_encode_char()
*
* Purpose:
*
* "Percent-encode" utf-8 character `c`, e.g.,
* '$' -> "%24"
* '¢' -> "%C2%A2"
*
* `c` cannot be null.
*
* Does not (currently) accept multi-byte characters...
* limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding
* (u+0080..u+07ff).
*
* Writes output to `repr`.
* `repr` cannot be null.
* Assumes adequate space i `repr`...
* >>> char[4] or [7] for most characters,
* >>> [13] as theoretical maximum.
*
* Representation `repr` is null-terminated.
*
* Stores length of representation (without null terminator) at pointer
* `repr_len`.
*
* Return : SUCCEED/FAIL
*
* - SUCCESS: `SUCCEED`
* - percent-encoded representation written to `repr`
* - 'repr' is null-terminated
* - FAILURE: `FAIL`
* - `c` or `repr` was NULL
*
* Programmer: Jacob Smith
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_percent_encode_char(
char *repr,
const unsigned char c,
size_t *repr_len)
{
unsigned int i = 0;
int chars_written = 0;
herr_t ret_value = SUCCEED;
#if S3COMMS_DEBUG
unsigned char s[2] = {c, 0};
unsigned char hex[3] = {0, 0, 0};
#endif
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_percent_encode_char.\n");
#endif
if (repr == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no destination `repr`.\n")
}
#if S3COMMS_DEBUG
H5FD_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE);
HDfprintf(stdout, " CHAR: \'%s\'\n", s);
HDfprintf(stdout, " CHAR-HEX: \"%s\"\n", hex);
#endif
if (c <= (unsigned char)0x7f) {
/* character represented in a single "byte"
* and single percent-code
*/
#if S3COMMS_DEBUG
HDfprintf(stdout, " SINGLE-BYTE\n");
#endif
*repr_len = 3;
chars_written = HDsnprintf(repr, 4, "%%%02X", c);
if (chars_written < 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot write char %c",
c);
}
} /* end if single-byte unicode char */
else {
/* multi-byte, multi-percent representation
*/
unsigned int acc = 0; /* byte accumulator */
unsigned int k = 0; /* uint character representation */
unsigned int stack_size = 0;
unsigned char stack[4] = {0, 0, 0, 0};
#if S3COMMS_DEBUG
HDfprintf(stdout, " MULTI-BYTE\n");
#endif
stack_size = 0;
k = (unsigned int)c;
*repr_len = 0;
do {
/* push number onto stack in six-bit slices
*/
acc = k;
acc >>= 6; /* cull least */
acc <<= 6; /* six bits */
stack[stack_size++] = (unsigned char)(k - acc);
k = acc >> 6;
} while (k > 0);
/* `stack` now has two to four six-bit 'numbers' to be put into
* UTF-8 byte fields.
*/
#if S3COMMS_DEBUG
HDfprintf(stdout, " STACK:\n {\n");
for (i = 0; i < stack_size; i++) {
H5FD_s3comms_bytes_to_hex(
(char *)hex,
(&stack[i]),
1,
FALSE);
hex[2] = 0;
HDfprintf(stdout, " %s,\n", hex);
}
HDfprintf(stdout, " }\n");
#endif
/****************
* leading byte *
****************/
/* prepend 11[1[1]]0 to first byte */
/* 110xxxxx, 1110xxxx, or 11110xxx */
acc = 0xC0; /* 0x11000000 */
acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */
acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */
stack_size--;
chars_written = HDsnprintf(
repr,
4,
"%%%02X",
(unsigned char)(acc + stack[stack_size]));
if (chars_written < 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot write char %c",
c);
}
*repr_len += 3;
/************************
* continuation byte(s) *
************************/
/* 10xxxxxx */
for (i = 0; i < stack_size; i++) {
chars_written = HDsnprintf(
&repr[i*3 + 3],
4,
"%%%02X",
(unsigned char)(0x80 + stack[stack_size - 1 - i]));
if (chars_written < 0) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"cannot write char %c",
c);
}
*repr_len += 3;
} /* end for each continuation byte */
} /* end else (multi-byte) */
*(repr + *repr_len) = '\0';
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_percent_encode_char */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_signing_key()
*
* Purpose:
*
* Create AWS4 "Signing Key" from secret key, AWS region, and timestamp.
*
* Sequentially runs HMAC_SHA256 on strings in specified order,
* generating re-usable checksum (according to documentation, valid for
* 7 days from time given).
*
* `secret` is `access key id` for targeted service/bucket/resource.
*
* `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z'
* e.g. "19690720T201740Z".
*
* `region` should be one of AWS service region names, e.g. "us-east-1".
*
* Hard-coded "service" algorithm requirement to "s3".
*
* Inputs must be null-terminated strings.
*
* Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`.
* Programmer must ensure that `md` is appropriately allocated.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - raw byte data of signing key written to `md`
* - FAILURE: `FAIL`
* - if any input arguments was NULL
*
* Programmer: Jacob Smith
* 2017-07-13
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_signing_key(
unsigned char *md,
const char *secret,
const char *region,
const char *iso8601now)
{
char *AWS4_secret = NULL;
size_t AWS4_secret_len = 0;
unsigned char datekey[SHA256_DIGEST_LENGTH];
unsigned char dateregionkey[SHA256_DIGEST_LENGTH];
unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH];
int ret = 0; /* return value of HDsnprintf */
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n");
#endif
if (md == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"Destination `md` cannot be NULL.\n")
}
if (secret == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"`secret` cannot be NULL.\n")
}
if (region == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"`region` cannot be NULL.\n")
}
if (iso8601now == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"`iso8601now` cannot be NULL.\n")
}
AWS4_secret_len = 4 + HDstrlen(secret) + 1;
AWS4_secret = (char*)H5MM_malloc(sizeof(char *) * AWS4_secret_len);
if (AWS4_secret == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"Could not allocate space.\n")
}
/* prepend "AWS4" to start of the secret key
*/
ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret);
if ((size_t)ret != (AWS4_secret_len - 1)) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem writing AWS4+secret `%s`",
secret);
}
/* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest
* we know digest length, so ignore via NULL
*/
HMAC(EVP_sha256(),
(const unsigned char *)AWS4_secret,
(int)HDstrlen(AWS4_secret),
(const unsigned char*)iso8601now,
8, /* 8 --> length of 8 --> "yyyyMMDD" */
datekey,
NULL);
HMAC(EVP_sha256(),
(const unsigned char *)datekey,
SHA256_DIGEST_LENGTH,
(const unsigned char *)region,
HDstrlen(region),
dateregionkey,
NULL);
HMAC(EVP_sha256(),
(const unsigned char *)dateregionkey,
SHA256_DIGEST_LENGTH,
(const unsigned char *)"s3",
2,
dateregionservicekey,
NULL);
HMAC(EVP_sha256(),
(const unsigned char *)dateregionservicekey,
SHA256_DIGEST_LENGTH,
(const unsigned char *)"aws4_request",
12,
md,
NULL);
done:
H5MM_xfree(AWS4_secret);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_signing_key() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_tostringtosign()
*
* Purpose:
*
* Get AWS "String to Sign" from Canonical Request, timestamp,
* and AWS "region".
*
* Common between single request and "chunked upload",
* conforms to:
* "AWS4-HMAC-SHA256\n" +
* <ISO8601 date format> + "\n" + // yyyyMMDD'T'hhmmss'Z'
* <yyyyMMDD> + "/" + <AWS Region> + "/s3/aws4-request\n" +
* hex(SHA256(<CANONICAL-REQUEST>))
*
* Inputs `creq` (canonical request string), `now` (ISO8601 format),
* and `region` (s3 region designator string) must all be
* null-terminated strings.
*
* Result is written to `dest` with null-terminator.
* It is left to programmer to ensure `dest` has adequate space.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - "string to sign" written to `dest` and null-terminated
* - FAILURE: `FAIL`
* - if any of the inputs are NULL
* - if an error is encountered while computing checksum
*
* Programmer: Jacob Smith
* 2017-07-??
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_tostringtosign(
char *dest,
const char *req,
const char *now,
const char *region)
{
unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1];
size_t d = 0;
char day[9];
char hexsum[SHA256_DIGEST_LENGTH * 2 + 1];
size_t i = 0;
int ret = 0; /* HDsnprintf return value */
herr_t ret_value = SUCCEED;
char tmp[128];
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n");
#endif
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"destination buffer cannot be null.\n")
}
if (req == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"canonical request cannot be null.\n")
}
if (now == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"Timestring cannot be NULL.\n")
}
if (region == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"Region cannot be NULL.\n")
}
for (i = 0; i < 128; i++) {
tmp[i] = '\0';
}
for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) {
checksum[i] = '\0';
hexsum[i] = '\0';
}
HDstrncpy(day, now, 8);
day[8] = '\0';
ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region);
if (ret <= 0 || ret >= 127) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"problem adding day and region to string")
}
HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17);
d = 17;
HDmemcpy((dest+d), now, HDstrlen(now));
d += HDstrlen(now);
dest[d++] = '\n';
HDmemcpy((dest + d), tmp, HDstrlen(tmp));
d += HDstrlen(tmp);
dest[d++] = '\n';
SHA256((const unsigned char *)req,
HDstrlen(req),
checksum);
if (H5FD_s3comms_bytes_to_hex(
hexsum,
(const unsigned char *)checksum,
SHA256_DIGEST_LENGTH,
true)
== FAIL)
{
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"could not create hex string");
}
for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++) {
dest[d++] = hexsum[i];
}
dest[d] = '\0';
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5ros3_tostringtosign() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_trim()
*
* Purpose:
*
* Remove all whitespace characters from start and end of a string `s`
* of length `s_len`, writing trimmed string copy to `dest`.
* Stores number of characters remaining at `n_written`.
*
* Destination for trimmed copy `dest` cannot be null.
* `dest` must have adequate space allocated for trimmed copy.
* If inadequate space, behavior is undefined, possibly resulting
* in segfault or overwrite of other data.
*
* If `s` is NULL or all whitespace, `dest` is untouched and `n_written`
* is set to 0.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* - `dest == NULL`
*
* Programmer: Jacob Smith
* 2017-09-18
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_trim(char *dest,
char *s,
size_t s_len,
size_t *n_written)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_trim.\n");
#endif
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"destination cannot be null.")
}
if (s == NULL) {
s_len = 0;
}
if (s_len > 0) {
/* Find first non-whitespace character from start;
* reduce total length per character.
*/
while ((s_len > 0) &&
isspace((unsigned char)s[0]) && s_len > 0)
{
s++;
s_len--;
}
/* Find first non-whitespace character from tail;
* reduce length per-character.
* If length is 0 already, there is no non-whitespace character.
*/
if (s_len > 0) {
do {
s_len--;
} while( isspace((unsigned char)s[s_len]) );
s_len++;
/* write output into dest
*/
HDmemcpy(dest, s, s_len);
}
}
*n_written = s_len;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_s3comms_trim() */
/*----------------------------------------------------------------------------
*
* Function: H5FD_s3comms_uriencode()
*
* Purpose:
*
* URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~".
*
* For each character in source string `_s` from `s[0]` to `s[s_len-1]`,
* writes to `dest` either the raw character or its percent-encoded
* equivalent.
*
* See `H5FD_s3comms_bytes_to_hex` for information on percent-encoding.
*
* Space (' ') character encoded as "%20" (not "+")
*
* Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`.
*
* Records number of characters written at `n_written`.
*
* Assumes that `dest` has been allocated with enough space.
*
* Neither `dest` nor `s` can be NULL.
*
* `s_len == 0` will have no effect.
*
* Return:
*
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* - source strings `s` or destination `dest` are NULL
* - error while attempting to percent-encode a character
*
* Programmer: Jacob Smith
* 2017-07-??
*
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_uriencode(
char *dest,
const char *s,
size_t s_len,
hbool_t encode_slash,
size_t *n_written)
{
char c = 0;
size_t dest_off = 0;
char hex_buffer[13];
size_t hex_off = 0;
size_t hex_len = 0;
herr_t ret_value = SUCCEED;
size_t s_off = 0;
FUNC_ENTER_NOAPI_NOINIT
#if S3COMMS_DEBUG
HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n");
#endif
if (s == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"source string cannot be NULL");
}
if (dest == NULL) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"destination cannot be NULL");
}
/* Write characters to destination, converting to percent-encoded
* "hex-utf-8" strings if necessary.
* e.g., '$' -> "%24"
*/
for (s_off = 0; s_off < s_len; s_off++) {
c = s[s_off];
if (isalnum(c) ||
c == '.' ||
c == '-' ||
c == '_' ||
c == '~' ||
(c == '/' && encode_slash == FALSE))
{
dest[dest_off++] = c;
}
else {
hex_off = 0;
if (H5FD_s3comms_percent_encode_char(
hex_buffer,
(const unsigned char)c,
&hex_len)
== FAIL)
{
hex_buffer[0] = c;
hex_buffer[1] = 0;
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to percent-encode character \'%s\' "
"at %d in \"%s\"", hex_buffer, (int)s_off, s);
}
for (hex_off = 0; hex_off < hex_len; hex_off++) {
dest[dest_off++] = hex_buffer[hex_off];
}
} /* end else (not a regular character) */
} /* end for each character */
if (dest_off < s_len)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"buffer overflow");
*n_written = dest_off;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD_s3comms_uriencode */
#endif /* H5_HAVE_ROS3_VFD */
Reference in New Issue View Git Blame Copy Permalink