Broken S3 implementation for netcdf files on non-AWS + Authenticated buckets

include/ncs3sdk.h

@@ -71,6 +71,7 @@ EXTERNL const char* NC_s3dumps3info(NCS3INFO* info);
 EXTERNL void NC_s3freeprofilelist(struct NClist* profiles);
 EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
 EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
+EXTERNL void NC_s3getcredentials(const char *profile, const char **region, const char **accessid, const char **accesskey);
 EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
 EXTERNL int NC_iss3(NCURI* uri, enum NCS3SVC*);
 EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);

libdispatch/dhttp.c

@@ -96,9 +96,12 @@ nc_http_open_verbose(const char* path, int verbose, NC_HTTP_STATE** statep)
     ncuriparse(path,&uri);
     if(uri == NULL) {stat = NCTHROW(NC_EURL); goto done;}
 
+    char *cleanpath = ncuribuild(uri, NULL, NULL, NCURISVC);
+    if(cleanpath == NULL) {stat = NCTHROW(NC_EURL); goto done;}
+
     if((state = calloc(1,sizeof(NC_HTTP_STATE))) == NULL)
         {stat = NCTHROW(NC_ENOMEM); goto done;}
-    state->path = strdup(path);
+    state->path = cleanpath;
     state->url = uri; uri = NULL;    
 #ifdef NETCDF_ENABLE_S3
     state->format = (NC_iss3(state->url,NULL)?HTTPS3:HTTPCURL);

libdispatch/dinfermodel.c

@@ -1342,7 +1342,7 @@ openmagic(struct MagicFile* file)
 	/* Construct a URL minus any fragment */
         file->curlurl = ncuribuild(file->uri,NULL,NULL,NCURISVC);
 	/* Open the curl handle */
-        if((status=nc_http_open(file->curlurl, &file->state))) goto done;
+        if((status=nc_http_open(file->path, &file->state))) goto done;
 	if((status=nc_http_size(file->state,&file->filelen))) goto done;
 #else /*!BYTERANGE*/
 	{status = NC_ENOTBUILT;}

libdispatch/ds3util.c

@@ -568,6 +568,34 @@ NC_s3profilelookup(const char* profile, const char* key, const char** valuep)
     if(valuep) *valuep = value;
     return stat;
 }
+/**
+ * Get the credentials for a given profile or load them from environment.
+ @param profile name to use to look for credentials
+ @param region return region from profile or env
+ @param accessid return accessid from progile or env
+ @param accesskey return accesskey from profile or env
+ */
+void NC_s3getcredentials(const char *profile, const char **region, const char** accessid, const char** accesskey) {
+    if(profile != NULL && strcmp(profile,"no") != 0) {
+        NC_s3profilelookup(profile, "aws_access_key_id", accessid);
+        NC_s3profilelookup(profile, "aws_secret_access_key", accesskey);
+        NC_s3profilelookup(profile, "region", region);
+    }
+    else
+    { // We load from env if not in profile
+        NCglobalstate* gstate = NC_getglobalstate();
+        if(gstate->aws.access_key_id != NULL && accessid){
+            *accessid = gstate->aws.access_key_id;
+        }
+        if (gstate->aws.secret_access_key != NULL && accesskey){
+            *accesskey = gstate->aws.secret_access_key;
+        }
+        if(gstate->aws.default_region != NULL && region){
+            *region = gstate->aws.default_region;
+        }
+    }
+}
+
 
 /**************************************************/
 /*

libdispatch/ncs3sdk_h5.c

@@ -182,10 +182,8 @@ NC_s3sdkcreateclient(NCS3INFO* info)
 
     s3client = (NCS3CLIENT*)calloc(1,sizeof(NCS3CLIENT));
     if(s3client == NULL) goto done;
-    if(info->profile != NULL) {
-        if((stat = NC_s3profilelookup(info->profile, "aws_access_key_id", &accessid))) goto done;
-        if((stat = NC_s3profilelookup(info->profile, "aws_secret_access_key", &accesskey))) goto done;
-    }
+    // We load credentials from env if not in profile
+    NC_s3getcredentials(info->profile, NULL, &accessid, &accesskey);
     if((s3client->rooturl = makes3rooturl(info))==NULL) {stat = NC_ENOMEM; goto done;}
     s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->svc,info->region,accessid,accesskey);
     if(s3client->h5s3client == NULL) {stat = NC_ES3; goto done;}

libnczarr/zmap_s3sdk.c

@@ -221,6 +221,9 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
         {stat = NC_EURL; goto done;}
 
     z3map->s3client = NC_s3sdkcreateclient(&z3map->s3);
+    if(z3map->s3client == NULL) {
+        stat = NC_ES3; goto done;
+    }
 
     /* Search the root for content */
     content = nclistnew();