From f07f5a80b753518c7a57c0ec8ce791fbfe77b25e Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 3 Nov 2023 23:23:41 +0800 Subject: [PATCH 01/14] refactor: notification.ts --- package-lock.json | 41 ++++++- package.json | 3 + src/minio.js | 4 +- src/notification.js | 201 --------------------------------- src/notification.ts | 265 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 205 deletions(-) delete mode 100644 src/notification.js create mode 100644 src/notification.ts diff --git a/package-lock.json b/package-lock.json index 58437108..4448756d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,19 @@ "version": "7.1.4", "license": "Apache-2.0", "dependencies": { + "@types/stream-json": "^1.7.5", "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^0.2.13", + "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.2.2", "ipaddr.js": "^2.0.1", "json-stream": "^1.0.0", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", + "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml": "^1.0.1", @@ -2254,8 +2257,7 @@ "node_modules/@types/node": { "version": "20.1.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz", - "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==", - "dev": true + "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==" }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -2269,6 +2271,23 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "node_modules/@types/stream-chain": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.3.tgz", + "integrity": "sha512-cwWE6mrdDpmW3B5wr1+vpjbg8h3hZfOr/PbKQ38VE21xNyF64GNjrh855YdNAPCt4kSYXuLwgRqBWkY/dD6KMg==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stream-json": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.5.tgz", + "integrity": "sha512-IVTtojYNqc6RT9FWBlwPLG6QTVdv2gHdqHOyBYPgcsCKfwIxcQpKw0e0ybQVyCvJJT9Le6w9RB5r5c6mo2m+IQ==", + "dependencies": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "node_modules/@types/through2": { "version": "2.0.38", "resolved": "https://registry.npmjs.org/@types/through2/-/through2-2.0.38.tgz", @@ -4183,6 +4202,11 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "node_modules/execa": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-7.1.1.tgz", @@ -7154,6 +7178,19 @@ "node": ">=6" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==" + }, + "node_modules/stream-json": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.8.0.tgz", + "integrity": "sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/strict-uri-encode": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", diff --git a/package.json b/package.json index 8e73cf98..373f0e79 100644 --- a/package.json +++ b/package.json @@ -82,16 +82,19 @@ }, "homepage": "https://github.com/minio/minio-js#readme", "dependencies": { + "@types/stream-json": "^1.7.5", "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^0.2.13", + "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.2.2", "ipaddr.js": "^2.0.1", "json-stream": "^1.0.0", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", + "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml": "^1.0.1", diff --git a/src/minio.js b/src/minio.js index 71cddf7d..54b45062 100644 --- a/src/minio.js +++ b/src/minio.js @@ -61,7 +61,7 @@ import { } from './internal/helper.ts' import { PostPolicy } from './internal/post-policy.ts' import { RETENTION_MODES, RETENTION_VALIDITY_UNITS } from './internal/type.ts' -import { NotificationConfig, NotificationPoller } from './notification.js' +import { NotificationConfig, NotificationPoller } from './notification.ts' import { ObjectUploader } from './object-uploader.js' import { promisify } from './promisify.js' import { postPresignSignatureV4, presignSignatureV4 } from './signing.ts' @@ -69,7 +69,7 @@ import * as transformers from './transformers.js' import { parseSelectObjectContentResponse } from './xml-parsers.js' export * from './helpers.ts' -export * from './notification.js' +export * from './notification.ts' export { CopyConditions, PostPolicy } export class Client extends TypedClient { diff --git a/src/notification.js b/src/notification.js deleted file mode 100644 index 2c3b45c3..00000000 --- a/src/notification.js +++ /dev/null @@ -1,201 +0,0 @@ -/* - * MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2016 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { EventEmitter } from 'node:events' - -import { DEFAULT_REGION } from './helpers.ts' -import { pipesetup, uriEscape } from './internal/helper.ts' -import * as transformers from './transformers.js' - -// Notification config - array of target configs. -// Target configs can be -// 1. Topic (simple notification service) -// 2. Queue (simple queue service) -// 3. CloudFront (lambda function) -export class NotificationConfig { - add(target) { - let instance = '' - if (target instanceof TopicConfig) { - instance = 'TopicConfiguration' - } - if (target instanceof QueueConfig) { - instance = 'QueueConfiguration' - } - if (target instanceof CloudFunctionConfig) { - instance = 'CloudFunctionConfiguration' - } - if (!this[instance]) { - this[instance] = [] - } - this[instance].push(target) - } -} - -// Base class for three supported configs. -class TargetConfig { - setId(id) { - this.Id = id - } - addEvent(newevent) { - if (!this.Event) { - this.Event = [] - } - this.Event.push(newevent) - } - addFilterSuffix(suffix) { - if (!this.Filter) { - this.Filter = { S3Key: { FilterRule: [] } } - } - this.Filter.S3Key.FilterRule.push({ Name: 'suffix', Value: suffix }) - } - addFilterPrefix(prefix) { - if (!this.Filter) { - this.Filter = { S3Key: { FilterRule: [] } } - } - this.Filter.S3Key.FilterRule.push({ Name: 'prefix', Value: prefix }) - } -} - -// 1. Topic (simple notification service) -export class TopicConfig extends TargetConfig { - constructor(arn) { - super() - this.Topic = arn - } -} - -// 2. Queue (simple queue service) -export class QueueConfig extends TargetConfig { - constructor(arn) { - super() - this.Queue = arn - } -} - -// 3. CloudFront (lambda function) -export class CloudFunctionConfig extends TargetConfig { - constructor(arn) { - super() - this.CloudFunction = arn - } -} - -export const buildARN = (partition, service, region, accountId, resource) => { - return 'arn:' + partition + ':' + service + ':' + region + ':' + accountId + ':' + resource -} - -export const ObjectCreatedAll = 's3:ObjectCreated:*' -export const ObjectCreatedPut = 's3:ObjectCreated:Put' -export const ObjectCreatedPost = 's3:ObjectCreated:Post' -export const ObjectCreatedCopy = 's3:ObjectCreated:Copy' -export const ObjectCreatedCompleteMultipartUpload = 's3:ObjectCreated:CompleteMultipartUpload' -export const ObjectRemovedAll = 's3:ObjectRemoved:*' -export const ObjectRemovedDelete = 's3:ObjectRemoved:Delete' -export const ObjectRemovedDeleteMarkerCreated = 's3:ObjectRemoved:DeleteMarkerCreated' -export const ObjectReducedRedundancyLostObject = 's3:ReducedRedundancyLostObject' - -// Poll for notifications, used in #listenBucketNotification. -// Listening constitutes repeatedly requesting s3 whether or not any -// changes have occurred. -export class NotificationPoller extends EventEmitter { - constructor(client, bucketName, prefix, suffix, events) { - super() - - this.client = client - this.bucketName = bucketName - this.prefix = prefix - this.suffix = suffix - this.events = events - - this.ending = false - } - - // Starts the polling. - start() { - this.ending = false - - process.nextTick(() => { - this.checkForChanges() - }) - } - - // Stops the polling. - stop() { - this.ending = true - } - - checkForChanges() { - // Don't continue if we're looping again but are cancelled. - if (this.ending) { - return - } - - let method = 'GET' - var queries = [] - if (this.prefix) { - var prefix = uriEscape(this.prefix) - queries.push(`prefix=${prefix}`) - } - if (this.suffix) { - var suffix = uriEscape(this.suffix) - queries.push(`suffix=${suffix}`) - } - if (this.events) { - this.events.forEach((s3event) => queries.push('events=' + uriEscape(s3event))) - } - queries.sort() - - var query = '' - if (queries.length > 0) { - query = `${queries.join('&')}` - } - const region = this.client.region || DEFAULT_REGION - this.client.makeRequest({ method, bucketName: this.bucketName, query }, '', [200], region, true, (e, response) => { - if (e) { - return this.emit('error', e) - } - - let transformer = transformers.getNotificationTransformer() - pipesetup(response, transformer) - .on('data', (result) => { - // Data is flushed periodically (every 5 seconds), so we should - // handle it after flushing from the JSON parser. - let records = result.Records - // If null (= no records), change to an empty array. - if (!records) { - records = [] - } - - // Iterate over the notifications and emit them individually. - records.forEach((record) => { - this.emit('notification', record) - }) - - // If we're done, stop. - if (this.ending) { - response.destroy() - } - }) - .on('error', (e) => this.emit('error', e)) - .on('end', () => { - // Do it again, if we haven't cancelled yet. - process.nextTick(() => { - this.checkForChanges() - }) - }) - }) - } -} diff --git a/src/notification.ts b/src/notification.ts new file mode 100644 index 00000000..0b0afdf6 --- /dev/null +++ b/src/notification.ts @@ -0,0 +1,265 @@ +/* + * MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2016 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EventEmitter } from 'eventemitter3' +import jsonLineParser from 'stream-json/jsonl/Parser.js' + +import { DEFAULT_REGION } from './helpers.ts' +import type { TypedClient } from './internal/client.ts' +import { pipesetup, uriEscape } from './internal/helper.ts' + +// TODO: type this + +type Event = unknown + +// Base class for three supported configs. +export class TargetConfig { + private Filter?: { S3Key: { FilterRule: { Name: string; Value: string }[] } } + private Event?: Event[] + private Id: unknown + + setId(id: unknown) { + this.Id = id + } + + addEvent(newevent: Event) { + if (!this.Event) { + this.Event = [] + } + this.Event.push(newevent) + } + + addFilterSuffix(suffix: string) { + if (!this.Filter) { + this.Filter = { S3Key: { FilterRule: [] } } + } + this.Filter.S3Key.FilterRule.push({ Name: 'suffix', Value: suffix }) + } + + addFilterPrefix(prefix: string) { + if (!this.Filter) { + this.Filter = { S3Key: { FilterRule: [] } } + } + this.Filter.S3Key.FilterRule.push({ Name: 'prefix', Value: prefix }) + } +} + +// 1. Topic (simple notification service) +export class TopicConfig extends TargetConfig { + private Topic: string + + constructor(arn: string) { + super() + this.Topic = arn + } +} + +// 2. Queue (simple queue service) +export class QueueConfig extends TargetConfig { + private Queue: string + + constructor(arn: string) { + super() + this.Queue = arn + } +} + +// 3. CloudFront (lambda function) +export class CloudFunctionConfig extends TargetConfig { + private CloudFunction: string + + constructor(arn: string) { + super() + this.CloudFunction = arn + } +} + +// Notification config - array of target configs. +// Target configs can be +// 1. Topic (simple notification service) +// 2. Queue (simple queue service) +// 3. CloudFront (lambda function) +export class NotificationConfig { + private TopicConfiguration?: TargetConfig[] + private CloudFunctionConfiguration?: TargetConfig[] + private QueueConfiguration?: TargetConfig[] + + add(target: TargetConfig) { + let instance: TargetConfig[] | undefined + if (target instanceof TopicConfig) { + instance = this.TopicConfiguration ??= [] + } + if (target instanceof QueueConfig) { + instance = this.QueueConfiguration ??= [] + } + if (target instanceof CloudFunctionConfig) { + instance = this.CloudFunctionConfiguration ??= [] + } + if (instance) { + instance.push(target) + } + } +} + +export const buildARN = (partition: string, service: string, region: string, accountId: string, resource: string) => { + return 'arn:' + partition + ':' + service + ':' + region + ':' + accountId + ':' + resource +} +export const ObjectCreatedAll = 's3:ObjectCreated:*' +export const ObjectCreatedPut = 's3:ObjectCreated:Put' +export const ObjectCreatedPost = 's3:ObjectCreated:Post' +export const ObjectCreatedCopy = 's3:ObjectCreated:Copy' +export const ObjectCreatedCompleteMultipartUpload = 's3:ObjectCreated:CompleteMultipartUpload' +export const ObjectRemovedAll = 's3:ObjectRemoved:*' +export const ObjectRemovedDelete = 's3:ObjectRemoved:Delete' +export const ObjectRemovedDeleteMarkerCreated = 's3:ObjectRemoved:DeleteMarkerCreated' +export const ObjectReducedRedundancyLostObject = 's3:ReducedRedundancyLostObject' +export type NotificationEvent = + | 's3:ObjectCreated:*' + | 's3:ObjectCreated:Put' + | 's3:ObjectCreated:Post' + | 's3:ObjectCreated:Copy' + | 's3:ObjectCreated:CompleteMultipartUpload' + | 's3:ObjectRemoved:*' + | 's3:ObjectRemoved:Delete' + | 's3:ObjectRemoved:DeleteMarkerCreated' + | 's3:ReducedRedundancyLostObject' + | 's3:TestEvent' + | 's3:ObjectRestore:Post' + | 's3:ObjectRestore:Completed' + | 's3:Replication:OperationFailedReplication' + | 's3:Replication:OperationMissedThreshold' + | 's3:Replication:OperationReplicatedAfterThreshold' + | 's3:Replication:OperationNotTracked' + | string // put string at least so auto-complete could work + +// TODO: type this +export type NotificationRecord = unknown +// Poll for notifications, used in #listenBucketNotification. +// Listening constitutes repeatedly requesting s3 whether or not any +// changes have occurred. +export class NotificationPoller extends EventEmitter<{ + notification: (event: NotificationRecord) => void + error: (error: unknown) => void +}> { + private client: TypedClient + private bucketName: string + private prefix: string + private suffix: string + private events: NotificationEvent[] + private ending: boolean + + constructor(client: TypedClient, bucketName: string, prefix: string, suffix: string, events: NotificationEvent[]) { + super() + + this.client = client + this.bucketName = bucketName + this.prefix = prefix + this.suffix = suffix + this.events = events + + this.ending = false + } + + // Starts the polling. + start() { + this.ending = false + + process.nextTick(() => { + this.checkForChanges() + }) + } + + // Stops the polling. + stop() { + this.ending = true + } + + checkForChanges() { + // Don't continue if we're looping again but are cancelled. + if (this.ending) { + return + } + + const method = 'GET' + const queries = [] + if (this.prefix) { + const prefix = uriEscape(this.prefix) + queries.push(`prefix=${prefix}`) + } + if (this.suffix) { + const suffix = uriEscape(this.suffix) + queries.push(`suffix=${suffix}`) + } + if (this.events) { + this.events.forEach((s3event) => queries.push('events=' + uriEscape(s3event))) + } + queries.sort() + + let query = '' + if (queries.length > 0) { + query = `${queries.join('&')}` + } + const region = this.client.region || DEFAULT_REGION + + this.client + .makeRequestAsync( + { + method, + bucketName: this.bucketName, + query, + }, + '', + [200], + region, + ) + .then( + (response) => { + const asm = jsonLineParser.make() + + pipesetup(response, asm) + .on('data', (data) => { + // Data is flushed periodically (every 5 seconds), so we should + // handle it after flushing from the JSON parser. + let records = data.value.Records + // If null (= no records), change to an empty array. + if (!records) { + records = [] + } + + // Iterate over the notifications and emit them individually. + records.forEach((record: NotificationRecord) => { + this.emit('notification', record) + }) + + // If we're done, stop. + if (this.ending) { + response?.destroy() + } + }) + .on('error', (e) => this.emit('error', e)) + .on('end', () => { + // Do it again, if we haven't cancelled yet. + process.nextTick(() => { + this.checkForChanges() + }) + }) + }, + (e) => { + return this.emit('error', e) + }, + ) + } +} From 89492cd8ee3cd8b078cfed2d2faf436ed6bbe231 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 3 Nov 2023 23:25:31 +0800 Subject: [PATCH 02/14] dep --- package-lock.json | 7 +++++-- package.json | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4448756d..a293ffe6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "7.1.4", "license": "Apache-2.0", "dependencies": { - "@types/stream-json": "^1.7.5", "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", @@ -38,6 +37,7 @@ "@types/lodash": "^4.14.194", "@types/mime-types": "^2.1.1", "@types/node": "^20.1.0", + "@types/stream-json": "^1.7.5", "@types/through2": "^2.0.38", "@types/xml": "^1.0.8", "@types/xml2js": "^0.4.11", @@ -2257,7 +2257,8 @@ "node_modules/@types/node": { "version": "20.1.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz", - "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==" + "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==", + "dev": true }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -2275,6 +2276,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.3.tgz", "integrity": "sha512-cwWE6mrdDpmW3B5wr1+vpjbg8h3hZfOr/PbKQ38VE21xNyF64GNjrh855YdNAPCt4kSYXuLwgRqBWkY/dD6KMg==", + "dev": true, "dependencies": { "@types/node": "*" } @@ -2283,6 +2285,7 @@ "version": "1.7.5", "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.5.tgz", "integrity": "sha512-IVTtojYNqc6RT9FWBlwPLG6QTVdv2gHdqHOyBYPgcsCKfwIxcQpKw0e0ybQVyCvJJT9Le6w9RB5r5c6mo2m+IQ==", + "dev": true, "dependencies": { "@types/node": "*", "@types/stream-chain": "*" diff --git a/package.json b/package.json index 373f0e79..ecc986e2 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,6 @@ }, "homepage": "https://github.com/minio/minio-js#readme", "dependencies": { - "@types/stream-json": "^1.7.5", "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", @@ -111,6 +110,7 @@ "@types/lodash": "^4.14.194", "@types/mime-types": "^2.1.1", "@types/node": "^20.1.0", + "@types/stream-json": "^1.7.5", "@types/through2": "^2.0.38", "@types/xml": "^1.0.8", "@types/xml2js": "^0.4.11", From 39fe88958f84dbe761d2a27bbeaec80cd0fcb7ef Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 3 Nov 2023 23:28:12 +0800 Subject: [PATCH 03/14] export --- src/minio.d.ts | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/src/minio.d.ts b/src/minio.d.ts index 13162a2b..d61742b9 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -44,9 +44,12 @@ import type { Tag, VersionIdentificator, } from './internal/type.ts' +import type { NotificationEvent } from './notification.ts' export * from './helpers.ts' export type { Region } from './internal/s3-endpoints.ts' +export type * from './notification.ts' +export * from './notification.ts' export { CopyConditions, PostPolicy } export type { BucketItem, @@ -79,26 +82,6 @@ export type { VersionIdentificator, } -// Exports only from typings -export type NotificationEvent = - | 's3:ObjectCreated:*' - | 's3:ObjectCreated:Put' - | 's3:ObjectCreated:Post' - | 's3:ObjectCreated:Copy' - | 's3:ObjectCreated:CompleteMultipartUpload' - | 's3:ObjectRemoved:*' - | 's3:ObjectRemoved:Delete' - | 's3:ObjectRemoved:DeleteMarkerCreated' - | 's3:ReducedRedundancyLostObject' - | 's3:TestEvent' - | 's3:ObjectRestore:Post' - | 's3:ObjectRestore:Completed' - | 's3:Replication:OperationFailedReplication' - | 's3:Replication:OperationMissedThreshold' - | 's3:Replication:OperationReplicatedAfterThreshold' - | 's3:Replication:OperationNotTracked' - | string - /** * @deprecated keep for backward compatible, use `RETENTION_MODES` instead */ @@ -561,13 +544,3 @@ export declare function buildARN( accountId: string, resource: string, ): string - -export declare const ObjectCreatedAll: NotificationEvent // s3:ObjectCreated:*' -export declare const ObjectCreatedPut: NotificationEvent // s3:ObjectCreated:Put -export declare const ObjectCreatedPost: NotificationEvent // s3:ObjectCreated:Post -export declare const ObjectCreatedCopy: NotificationEvent // s3:ObjectCreated:Copy -export declare const ObjectCreatedCompleteMultipartUpload: NotificationEvent // s3:ObjectCreated:CompleteMultipartUpload -export declare const ObjectRemovedAll: NotificationEvent // s3:ObjectRemoved:* -export declare const ObjectRemovedDelete: NotificationEvent // s3:ObjectRemoved:Delete -export declare const ObjectRemovedDeleteMarkerCreated: NotificationEvent // s3:ObjectRemoved:DeleteMarkerCreated -export declare const ObjectReducedRedundancyLostObject: NotificationEvent // s3:ReducedRedundancyLostObject From 834f1fc518be291cced02bf603c9c3d79fb52e49 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 3 Nov 2023 23:34:55 +0800 Subject: [PATCH 04/14] move to internal --- src/{ => internal}/notification.ts | 6 ++--- src/minio.d.ts | 40 +++--------------------------- src/minio.js | 4 +-- 3 files changed, 8 insertions(+), 42 deletions(-) rename src/{ => internal}/notification.ts (97%) diff --git a/src/notification.ts b/src/internal/notification.ts similarity index 97% rename from src/notification.ts rename to src/internal/notification.ts index 0b0afdf6..e42e7ad6 100644 --- a/src/notification.ts +++ b/src/internal/notification.ts @@ -17,9 +17,9 @@ import { EventEmitter } from 'eventemitter3' import jsonLineParser from 'stream-json/jsonl/Parser.js' -import { DEFAULT_REGION } from './helpers.ts' -import type { TypedClient } from './internal/client.ts' -import { pipesetup, uriEscape } from './internal/helper.ts' +import { DEFAULT_REGION } from '../helpers.ts' +import type { TypedClient } from './client.ts' +import { pipesetup, uriEscape } from './helper.ts' // TODO: type this diff --git a/src/minio.d.ts b/src/minio.d.ts index d61742b9..b1b1ea94 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -1,7 +1,6 @@ // imported from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/93cfb0ec069731dcdfc31464788613f7cddb8192/types/minio/index.d.ts /* eslint-disable @typescript-eslint/no-explicit-any */ -import { EventEmitter } from 'node:events' import type { Readable as ReadableStream } from 'node:stream' import type { @@ -14,6 +13,7 @@ import type { import type { ClientOptions, NoResultCallback, RemoveOptions } from './internal/client.ts' import { TypedClient } from './internal/client.ts' import { CopyConditions } from './internal/copy-conditions.ts' +import type { NotificationConfig, NotificationEvent, NotificationPoller } from './internal/notification.ts' import { PostPolicy } from './internal/post-policy.ts' import type { Region } from './internal/s3-endpoints.ts' import type { @@ -44,12 +44,11 @@ import type { Tag, VersionIdentificator, } from './internal/type.ts' -import type { NotificationEvent } from './notification.ts' export * from './helpers.ts' +export type * from './internal/notification.ts' +export * from './internal/notification.ts' export type { Region } from './internal/s3-endpoints.ts' -export type * from './notification.ts' -export * from './notification.ts' export { CopyConditions, PostPolicy } export type { BucketItem, @@ -511,36 +510,3 @@ export class Client extends TypedClient { // Other newPostPolicy(): PostPolicy } - -export declare class NotificationPoller extends EventEmitter { - stop(): void - - start(): void - - // must to be public? - checkForChanges(): void -} - -export declare class NotificationConfig { - add(target: TopicConfig | QueueConfig | CloudFunctionConfig): void -} - -export declare class TopicConfig extends TargetConfig { - constructor(arn: string) -} - -export declare class QueueConfig extends TargetConfig { - constructor(arn: string) -} - -export declare class CloudFunctionConfig extends TargetConfig { - constructor(arn: string) -} - -export declare function buildARN( - partition: string, - service: string, - region: string, - accountId: string, - resource: string, -): string diff --git a/src/minio.js b/src/minio.js index 54b45062..734d2bc2 100644 --- a/src/minio.js +++ b/src/minio.js @@ -59,9 +59,9 @@ import { uriEscape, uriResourceEscape, } from './internal/helper.ts' +import { NotificationConfig, NotificationPoller } from './internal/notification.ts' import { PostPolicy } from './internal/post-policy.ts' import { RETENTION_MODES, RETENTION_VALIDITY_UNITS } from './internal/type.ts' -import { NotificationConfig, NotificationPoller } from './notification.ts' import { ObjectUploader } from './object-uploader.js' import { promisify } from './promisify.js' import { postPresignSignatureV4, presignSignatureV4 } from './signing.ts' @@ -69,7 +69,7 @@ import * as transformers from './transformers.js' import { parseSelectObjectContentResponse } from './xml-parsers.js' export * from './helpers.ts' -export * from './notification.ts' +export * from './internal/notification.ts' export { CopyConditions, PostPolicy } export class Client extends TypedClient { From 072aae32b8ed344b2002c429ed65fdfbd134359b Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 5 Nov 2023 18:09:40 +0800 Subject: [PATCH 05/14] Update minio.d.ts --- src/minio.d.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/minio.d.ts b/src/minio.d.ts index 841fb6f8..d18f3323 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -28,6 +28,7 @@ import type { IncompleteUploadedBucketItem, ItemBucketMetadata, ItemBucketMetadataList, + LegalHoldStatus, MetadataItem, ObjectLockInfo, PutObjectLegalHoldOptions, @@ -62,6 +63,7 @@ export type { IncompleteUploadedBucketItem, ItemBucketMetadata, ItemBucketMetadataList, + LegalHoldStatus, MetadataItem, NoResultCallback, ObjectLockInfo, @@ -89,10 +91,6 @@ export type Mode = RETENTION_MODES */ export type LockUnit = RETENTION_VALIDITY_UNITS -/** - * @deprecated keep for backward compatible - */ -export type LegalHoldStatus = LEGAL_HOLD_STATUS export type VersioningConfig = Record export type TagList = Record export type EmptyObject = Record From 4ccb312817a14bd66c9250de3d1f0ddff6c50e4d Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 5 Nov 2023 18:10:00 +0800 Subject: [PATCH 06/14] Update minio.d.ts --- src/minio.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/minio.d.ts b/src/minio.d.ts index d18f3323..da10690d 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -94,7 +94,7 @@ export type LockUnit = RETENTION_VALIDITY_UNITS export type VersioningConfig = Record export type TagList = Record export type EmptyObject = Record -export type VersionIdentificator = Pick +export type VersionIdentificator = { versionId: string } export type Lifecycle = LifecycleConfig | null | '' export type Encryption = EncryptionConfig | EmptyObject export type Retention = RetentionOptions | EmptyObject From 3bf4bd768f531a10e67190130a94fcc63ee69454 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 14 Nov 2023 23:14:53 +0800 Subject: [PATCH 07/14] Refactor notification polling request. --- src/internal/notification.ts | 79 ++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/src/internal/notification.ts b/src/internal/notification.ts index e42e7ad6..4957c047 100644 --- a/src/internal/notification.ts +++ b/src/internal/notification.ts @@ -214,52 +214,41 @@ export class NotificationPoller extends EventEmitter<{ } const region = this.client.region || DEFAULT_REGION - this.client - .makeRequestAsync( - { - method, - bucketName: this.bucketName, - query, - }, - '', - [200], - region, - ) - .then( - (response) => { - const asm = jsonLineParser.make() - - pipesetup(response, asm) - .on('data', (data) => { - // Data is flushed periodically (every 5 seconds), so we should - // handle it after flushing from the JSON parser. - let records = data.value.Records - // If null (= no records), change to an empty array. - if (!records) { - records = [] - } - - // Iterate over the notifications and emit them individually. - records.forEach((record: NotificationRecord) => { - this.emit('notification', record) - }) - - // If we're done, stop. - if (this.ending) { - response?.destroy() - } + this.client.makeRequestAsync({ method, bucketName: this.bucketName, query }, '', [200], region).then( + (response) => { + const asm = jsonLineParser.make() + + pipesetup(response, asm) + .on('data', (data) => { + // Data is flushed periodically (every 5 seconds), so we should + // handle it after flushing from the JSON parser. + let records = data.value.Records + // If null (= no records), change to an empty array. + if (!records) { + records = [] + } + + // Iterate over the notifications and emit them individually. + records.forEach((record: NotificationRecord) => { + this.emit('notification', record) }) - .on('error', (e) => this.emit('error', e)) - .on('end', () => { - // Do it again, if we haven't cancelled yet. - process.nextTick(() => { - this.checkForChanges() - }) + + // If we're done, stop. + if (this.ending) { + response?.destroy() + } + }) + .on('error', (e) => this.emit('error', e)) + .on('end', () => { + // Do it again, if we haven't cancelled yet. + process.nextTick(() => { + this.checkForChanges() }) - }, - (e) => { - return this.emit('error', e) - }, - ) + }) + }, + (e) => { + return this.emit('error', e) + }, + ) } } From 04093946110af0fcd23352800708c120fcf5a88f Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 14 Nov 2023 23:17:12 +0800 Subject: [PATCH 08/14] Update NotificationConfig import and export in minio.js and minio.d.ts --- src/minio.d.ts | 6 +++--- src/minio.js | 4 ++-- src/{internal => }/notification.ts | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename src/{internal => }/notification.ts (97%) diff --git a/src/minio.d.ts b/src/minio.d.ts index f6df9e15..545c5c55 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -13,7 +13,6 @@ import type { import type { ClientOptions, NoResultCallback, RemoveOptions } from './internal/client.ts' import { TypedClient } from './internal/client.ts' import { CopyConditions } from './internal/copy-conditions.ts' -import type { NotificationConfig, NotificationEvent, NotificationPoller } from './internal/notification.ts' import { PostPolicy } from './internal/post-policy.ts' import type { BucketItem, @@ -48,11 +47,12 @@ import type { Tag, VersionIdentificator, } from './internal/type.ts' +import type { NotificationConfig, NotificationEvent, NotificationPoller } from './notification.ts' export * from './helpers.ts' -export type * from './internal/notification.ts' -export * from './internal/notification.ts' export type { Region } from './internal/s3-endpoints.ts' +export type * from './notification.ts' +export * from './notification.ts' export { CopyConditions, PostPolicy } export type { MakeBucketOpt } from './internal/client.ts' export type { diff --git a/src/minio.js b/src/minio.js index 11832d33..9d4f2a96 100644 --- a/src/minio.js +++ b/src/minio.js @@ -58,8 +58,8 @@ import { uriEscape, uriResourceEscape, } from './internal/helper.ts' -import { NotificationConfig, NotificationPoller } from './internal/notification.ts' import { PostPolicy } from './internal/post-policy.ts' +import { NotificationConfig, NotificationPoller } from './notification.ts' import { ObjectUploader } from './object-uploader.js' import { promisify } from './promisify.js' import { postPresignSignatureV4, presignSignatureV4 } from './signing.ts' @@ -67,7 +67,7 @@ import * as transformers from './transformers.js' import { parseSelectObjectContentResponse } from './xml-parsers.js' export * from './helpers.ts' -export * from './internal/notification.ts' +export * from './notification.ts' export { CopyConditions, PostPolicy } export class Client extends TypedClient { diff --git a/src/internal/notification.ts b/src/notification.ts similarity index 97% rename from src/internal/notification.ts rename to src/notification.ts index 4957c047..64c24112 100644 --- a/src/internal/notification.ts +++ b/src/notification.ts @@ -17,9 +17,9 @@ import { EventEmitter } from 'eventemitter3' import jsonLineParser from 'stream-json/jsonl/Parser.js' -import { DEFAULT_REGION } from '../helpers.ts' -import type { TypedClient } from './client.ts' -import { pipesetup, uriEscape } from './helper.ts' +import { DEFAULT_REGION } from './helpers.ts' +import type { TypedClient } from './internal/client.ts' +import { pipesetup, uriEscape } from './internal/helper.ts' // TODO: type this From 5081e5a10080842d8f994482d96321a9b4c6764d Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 28 Nov 2023 13:24:10 +0800 Subject: [PATCH 09/14] Update src/minio.js --- src/minio.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/minio.js b/src/minio.js index 72c01fe6..b6b6ee77 100644 --- a/src/minio.js +++ b/src/minio.js @@ -56,7 +56,6 @@ import { import { PostPolicy } from './internal/post-policy.ts' import { NotificationConfig, NotificationPoller } from './notification.ts' -import { ObjectUploader } from './object-uploader.js' import { promisify } from './promisify.js' import { postPresignSignatureV4, presignSignatureV4 } from './signing.ts' From 8f03c47ce3cba0f610f0229106e81306848dd38a Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 28 Nov 2023 13:25:07 +0800 Subject: [PATCH 10/14] Update src/minio.js --- src/minio.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/minio.js b/src/minio.js index b6b6ee77..2a9b2e43 100644 --- a/src/minio.js +++ b/src/minio.js @@ -54,7 +54,6 @@ import { uriResourceEscape, } from './internal/helper.ts' import { PostPolicy } from './internal/post-policy.ts' - import { NotificationConfig, NotificationPoller } from './notification.ts' import { promisify } from './promisify.js' From 7ee19f7a2e4c824a814379f6895714ce846c6792 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 28 Nov 2023 13:25:33 +0800 Subject: [PATCH 11/14] Update src/minio.js --- src/minio.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/minio.js b/src/minio.js index 2a9b2e43..b9d385a6 100644 --- a/src/minio.js +++ b/src/minio.js @@ -55,7 +55,6 @@ import { } from './internal/helper.ts' import { PostPolicy } from './internal/post-policy.ts' import { NotificationConfig, NotificationPoller } from './notification.ts' - import { promisify } from './promisify.js' import { postPresignSignatureV4, presignSignatureV4 } from './signing.ts' import * as transformers from './transformers.js' From 210836e943608c7b8b5804ed02499aec056db509 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 28 Nov 2023 16:32:47 +0800 Subject: [PATCH 12/14] Remove json-stream dependency --- package-lock.json | 7 ------- package.json | 1 - src/transformers.js | 7 ------- 3 files changed, 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index 587eeb7e..3cc8ba75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.2.2", "ipaddr.js": "^2.0.1", - "json-stream": "^1.0.0", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", @@ -5409,12 +5408,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", - "integrity": "sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==", - "license": "MIT" - }, "node_modules/json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", diff --git a/package.json b/package.json index 6685d28d..a2be2e75 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,6 @@ "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.2.2", "ipaddr.js": "^2.0.1", - "json-stream": "^1.0.0", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", diff --git a/src/transformers.js b/src/transformers.js index 4ad168f6..f10945d5 100644 --- a/src/transformers.js +++ b/src/transformers.js @@ -16,7 +16,6 @@ import * as Crypto from 'node:crypto' -import JSONParser from 'json-stream' import Through2 from 'through2' import { isFunction } from './internal/helper.ts' @@ -121,12 +120,6 @@ export function getBucketNotificationTransformer() { return getConcater(xmlParsers.parseBucketNotification) } -// Parses a notification. -export function getNotificationTransformer() { - // This will parse and return each object. - return new JSONParser() -} - export function lifecycleTransformer() { return getConcater(xmlParsers.parseLifecycleConfig) } From 43f86b3ca35244bbae1a43229851bab1dac18b1c Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 8 Dec 2023 15:26:51 +0800 Subject: [PATCH 13/14] Update src/minio.d.ts --- src/minio.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/minio.d.ts b/src/minio.d.ts index ee70ddef..33ce92d0 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -1,7 +1,7 @@ // imported from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/93cfb0ec069731dcdfc31464788613f7cddb8192/types/minio/index.d.ts /* eslint-disable @typescript-eslint/no-explicit-any */ -import type { Readable as ReadableStream } from 'node:stream' +import { EventEmitter } from 'node:events' import type { CopyDestinationOptions, From b45c658e10ba7f58b3ab8399cbe0f2a9d9d481f2 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 8 Dec 2023 15:30:18 +0800 Subject: [PATCH 14/14] Update minio.d.ts --- src/minio.d.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/minio.d.ts b/src/minio.d.ts index 33ce92d0..d0decce6 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -1,8 +1,6 @@ // imported from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/93cfb0ec069731dcdfc31464788613f7cddb8192/types/minio/index.d.ts /* eslint-disable @typescript-eslint/no-explicit-any */ -import { EventEmitter } from 'node:events' - import type { CopyDestinationOptions, CopySourceOptions,