Skip to content

Commit

Permalink
[potential fixup] use centralised s3 functions
Browse files Browse the repository at this point in the history
  • Loading branch information
jameshadfield committed Nov 7, 2023
1 parent e9231cb commit 0b56c84
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions src/resourceIndex.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ import fs from 'fs';
import * as utils from './utils/index.js';
import zlib from 'zlib';
import { promisify } from 'util';
import AWS from 'aws-sdk';
import { NotFound, BadRequest } from './httpErrors.js';
import { RESOURCE_INDEX } from './config.js';
import { signedUrl } from './s3.js';
import { fetch } from './fetch.js';

const gunzip = promisify(zlib.gunzip)

Expand Down Expand Up @@ -100,18 +101,14 @@ async function updateResourceVersions() {
} else {
const [Bucket, Key] = ['nextstrain-inventories', 'resources.json.gz'];
utils.verbose(`Updating available resources index from s3://${Bucket}/${Key}`);
const S3 = new AWS.S3();
try {
const newETag = (await S3.headObject({Bucket, Key}).promise()).ETag;
const newETag = (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'HEAD'}), {method: 'HEAD'}))
.headers.get('etag'); // value is enclosed in double quotes, but that doesn't matter for our purposes
if (newETag && newETag === eTag) {
utils.verbose("Skipping available resource update as eTag hasn't changed");
return;
}
/* NOTE: in AWS SDK v2 response.Body is a buffer; in v3 it's a stream */
const newResources = JSON.parse(
await S3.getObject({Bucket, Key}).promise()
.then((response) => gunzip(response.Body))
);
const newResources = JSON.parse(await gunzip(await (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'GET'}), {method: 'GET'})).buffer()));
[resources, eTag] = [newResources, newETag];
} catch (err) {
utils.warn(`Resource updating failed: ${err.message}`)
Expand Down

0 comments on commit 0b56c84

Please sign in to comment.