diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 972a8c7ea..24943e116 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest - digest: sha256:8060aba1f6d5617d08091767141ab2a99ea1ccbd9371fd42ffc208c5329caa73 -# created: 2024-10-04T00:09:31.22876583Z + digest: sha256:609822e3c09b7a1bd90b99655904609f162cc15acb4704f1edf778284c36f429 +# created: 2024-10-01T19:34:30.797530443Z diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9901e14cb..4892eb2c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: [14, 16, 18}] + node: [14, 16, 18, 20] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: ${{ matrix.node }} - run: node --version @@ -29,10 +29,10 @@ jobs: windows: runs-on: windows-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 14 - run: npm install --engine-strict - run: npm test env: @@ -40,19 +40,19 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 14 - run: npm install - run: npm run lint docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 14 - run: npm install - run: npm run docs - uses: JustinBeckwith/linkinator-action@v1 diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg index dabc141b2..d8038cd99 100644 --- a/.kokoro/common.cfg +++ b/.kokoro/common.cfg @@ -16,7 +16,7 @@ build_file: "nodejs-pubsub/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" } env_vars: { key: "TRAMPOLINE_BUILD_FILE" diff --git a/.kokoro/continuous/node18/common.cfg b/.kokoro/continuous/node18/common.cfg deleted file mode 100644 index dabc141b2..000000000 --- a/.kokoro/continuous/node18/common.cfg +++ /dev/null @@ -1,24 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "nodejs-pubsub/.kokoro/trampoline_v2.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/test.sh" -} diff --git a/.kokoro/continuous/node18/lint.cfg b/.kokoro/continuous/node18/lint.cfg deleted file mode 100644 index 219d3808d..000000000 --- a/.kokoro/continuous/node18/lint.cfg +++ /dev/null @@ -1,4 +0,0 @@ -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/lint.sh" -} diff --git a/.kokoro/continuous/node18/samples-test.cfg b/.kokoro/continuous/node18/samples-test.cfg deleted file mode 100644 index 698013d7c..000000000 --- a/.kokoro/continuous/node18/samples-test.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/samples-test.sh" -} - -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" -} \ No newline at end of file diff --git a/.kokoro/continuous/node18/system-test.cfg b/.kokoro/continuous/node18/system-test.cfg deleted file mode 100644 index bf759bd3f..000000000 --- a/.kokoro/continuous/node18/system-test.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/system-test.sh" -} - -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" -} \ No newline at end of file diff --git a/.kokoro/continuous/node18/test.cfg b/.kokoro/continuous/node18/test.cfg deleted file mode 100644 index e69de29bb..000000000 diff --git a/.kokoro/presubmit/node18/common.cfg b/.kokoro/presubmit/node18/common.cfg deleted file mode 100644 index dabc141b2..000000000 --- a/.kokoro/presubmit/node18/common.cfg +++ /dev/null @@ -1,24 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "nodejs-pubsub/.kokoro/trampoline_v2.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/test.sh" -} diff --git a/.kokoro/presubmit/node18/samples-test.cfg b/.kokoro/presubmit/node18/samples-test.cfg deleted file mode 100644 index 698013d7c..000000000 --- a/.kokoro/presubmit/node18/samples-test.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/samples-test.sh" -} - -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" -} \ No newline at end of file diff --git a/.kokoro/presubmit/node18/system-test.cfg b/.kokoro/presubmit/node18/system-test.cfg deleted file mode 100644 index bf759bd3f..000000000 --- a/.kokoro/presubmit/node18/system-test.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/nodejs-pubsub/.kokoro/system-test.sh" -} - -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" -} \ No newline at end of file diff --git a/.kokoro/presubmit/node18/test.cfg b/.kokoro/presubmit/node18/test.cfg deleted file mode 100644 index e69de29bb..000000000 diff --git a/.kokoro/release/docs-devsite.cfg b/.kokoro/release/docs-devsite.cfg index b1ebf97bc..b5638214a 100644 --- a/.kokoro/release/docs-devsite.cfg +++ b/.kokoro/release/docs-devsite.cfg @@ -11,7 +11,7 @@ before_action { # doc publications use a Python image. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" } # Download trampoline resources. diff --git a/.kokoro/release/docs.cfg b/.kokoro/release/docs.cfg index ef7147c33..508a0efda 100644 --- a/.kokoro/release/docs.cfg +++ b/.kokoro/release/docs.cfg @@ -11,7 +11,7 @@ before_action { # doc publications use a Python image. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" } # Download trampoline resources. diff --git a/.kokoro/release/docs.sh b/.kokoro/release/docs.sh index e9079a605..1d8f3f490 100755 --- a/.kokoro/release/docs.sh +++ b/.kokoro/release/docs.sh @@ -16,7 +16,7 @@ set -eo pipefail -# build jsdocs (Python is installed on the Node 18 docker image). +# build jsdocs (Python is installed on the Node 10 docker image). if [[ -z "$CREDENTIALS" ]]; then # if CREDENTIALS are explicitly set, assume we're testing locally # and don't set NPM_CONFIG_PREFIX. diff --git a/.kokoro/release/publish.cfg b/.kokoro/release/publish.cfg index ae6223c88..91fb161ec 100644 --- a/.kokoro/release/publish.cfg +++ b/.kokoro/release/publish.cfg @@ -30,7 +30,7 @@ build_file: "nodejs-pubsub/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" } env_vars: { diff --git a/.kokoro/samples-test.sh b/.kokoro/samples-test.sh index c1cb0fc77..8c5d108cb 100755 --- a/.kokoro/samples-test.sh +++ b/.kokoro/samples-test.sh @@ -56,7 +56,7 @@ fi # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=18 +COVERAGE_NODE=14 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/.kokoro/system-test.sh b/.kokoro/system-test.sh index a90d5cfec..0b3043d26 100755 --- a/.kokoro/system-test.sh +++ b/.kokoro/system-test.sh @@ -49,7 +49,7 @@ npm run system-test # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=18 +COVERAGE_NODE=14 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/.kokoro/test.bat b/.kokoro/test.bat index caf825656..0bb124052 100644 --- a/.kokoro/test.bat +++ b/.kokoro/test.bat @@ -21,7 +21,7 @@ cd .. @rem we upgrade Node.js in the image: SET PATH=%PATH%;/cygdrive/c/Program Files/nodejs/npm -call nvm use 18 +call nvm use v14.17.3 call which node call npm install || goto :error diff --git a/.kokoro/test.sh b/.kokoro/test.sh index 0d9f6392a..862d478d3 100755 --- a/.kokoro/test.sh +++ b/.kokoro/test.sh @@ -39,7 +39,7 @@ npm test # codecov combines coverage across integration and unit tests. Include # the logic below for any environment you wish to collect coverage for: -COVERAGE_NODE=18 +COVERAGE_NODE=14 if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then NYC_BIN=./node_modules/nyc/bin/nyc.js if [ -f "$NYC_BIN" ]; then diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 5d6cfcca5..4d0311212 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -44,7 +44,7 @@ # the project root. # # Here is an example for running this script. -# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:18-user \ +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ # TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ # .kokoro/trampoline_v2.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index c0f5aecdd..96023ed8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://www.npmjs.com/package/@google-cloud/pubsub?activeTab=versions +## [4.8.0](https://github.com/googleapis/nodejs-pubsub/compare/v4.7.2...v4.8.0) (2024-10-15) + + +### Features + +* Add ingestion Cloud Storage fields and Platform Logging fields to Topic ([#1974](https://github.com/googleapis/nodejs-pubsub/issues/1974)) ([afec9a1](https://github.com/googleapis/nodejs-pubsub/commit/afec9a1ad3f665a71f08e748623f0fdaa332d17b)) +* Return listing information for subscriptions created via Analytics Hub ([afec9a1](https://github.com/googleapis/nodejs-pubsub/commit/afec9a1ad3f665a71f08e748623f0fdaa332d17b)) + ## [4.7.2](https://github.com/googleapis/nodejs-pubsub/compare/v4.7.1...v4.7.2) (2024-09-13) diff --git a/README.md b/README.md index 210ec6d79..db9c9ef33 100644 --- a/README.md +++ b/README.md @@ -66,15 +66,15 @@ npm install @google-cloud/pubsub ```javascript // Imports the Google Cloud client library -const { PubSub } = require("@google-cloud/pubsub"); +const {PubSub} = require('@google-cloud/pubsub'); async function quickstart( -projectId = 'your-project-id', // Your Google Cloud Platform project ID -topicNameOrId = 'my-topic', // Name for the new topic to create -subscriptionName = 'my-sub' // Name for the new subscription to create + projectId = 'your-project-id', // Your Google Cloud Platform project ID + topicNameOrId = 'my-topic', // Name for the new topic to create + subscriptionName = 'my-sub' // Name for the new subscription to create ) { // Instantiates a client - const pubsub = new PubSub({ projectId }); + const pubsub = new PubSub({projectId}); // Creates a new topic const [topic] = await pubsub.createTopic(topicNameOrId); @@ -84,19 +84,19 @@ subscriptionName = 'my-sub' // Name for the new subscription to create const [subscription] = await topic.createSubscription(subscriptionName); // Receive callbacks for new messages on the subscription - subscription.on('message', (message) => { + subscription.on('message', message => { console.log('Received message:', message.data.toString()); process.exit(0); }); // Receive callbacks for errors on the subscription - subscription.on('error', (error) => { + subscription.on('error', error => { console.error('Received error:', error); process.exit(1); }); // Send a message to the topic - topic.publishMessage({ data: Buffer.from('Test message!') }); + topic.publishMessage({data: Buffer.from('Test message!')}); } ``` @@ -138,6 +138,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-pubsub/tree | Create Subscription with ordering enabled | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createSubscriptionWithOrdering.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createSubscriptionWithOrdering.js,samples/README.md) | | Create Subscription With Retry Policy | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createSubscriptionWithRetryPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createSubscriptionWithRetryPolicy.js,samples/README.md) | | Create Topic | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopic.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopic.js,samples/README.md) | +| Create Topic With Cloud Storage Ingestion | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopicWithCloudStorageIngestion.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopicWithCloudStorageIngestion.js,samples/README.md) | | Create Topic With Kinesis Ingestion | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopicWithKinesisIngestion.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopicWithKinesisIngestion.js,samples/README.md) | | Create Topic With Schema | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopicWithSchema.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopicWithSchema.js,samples/README.md) | | Create Topic With Schema Revisions | [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopicWithSchemaRevisions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopicWithSchemaRevisions.js,samples/README.md) | diff --git a/package.json b/package.json index b3558bf47..d9bbaa757 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/pubsub", "description": "Cloud Pub/Sub Client Library for Node.js", - "version": "4.7.2", + "version": "4.8.0", "license": "Apache-2.0", "author": "Google Inc.", "engines": { @@ -29,7 +29,8 @@ "scripts": { "presystem-test": "npm run compile", "system-test": "mocha build/system-test --timeout 600000", - "samples-test": "cd samples/ && npm link ../ && npm install && npm test && cd ../", + "samples-test": "npm run link-samples && npm test && cd ../", + "link-samples": "cd samples/ && npm link ../ && npm install", "test": "c8 mocha build/test --recursive", "lint": "gts check", "predocs": "npm run compile", @@ -46,7 +47,7 @@ "prelint": "cd samples; npm link ../; npm install", "precompile": "gts clean", "typeless": "npx typeless-sample-bot --outputpath samples --targets samples --recursive", - "posttypeless": "cd samples && npm i && cd .. && npx eslint --ignore-pattern owl-bot-staging --fix" + "posttypeless": "npm run link-samples && npx eslint --ignore-pattern owl-bot-staging --fix samples" }, "dependencies": { "@google-cloud/paginator": "^5.0.0", diff --git a/samples/README.md b/samples/README.md index c3e17d913..dc6caf537 100644 --- a/samples/README.md +++ b/samples/README.md @@ -35,6 +35,7 @@ guides. * [Create Subscription with ordering enabled](#create-subscription-with-ordering-enabled) * [Create Subscription With Retry Policy](#create-subscription-with-retry-policy) * [Create Topic](#create-topic) + * [Create Topic With Cloud Storage Ingestion](#create-topic-with-cloud-storage-ingestion) * [Create Topic With Kinesis Ingestion](#create-topic-with-kinesis-ingestion) * [Create Topic With Schema](#create-topic-with-schema) * [Create Topic With Schema Revisions](#create-topic-with-schema-revisions) @@ -389,6 +390,25 @@ __Usage:__ +### Create Topic With Cloud Storage Ingestion + +Creates a new topic, with Cloud Storage ingestion enabled. + +View the [source code](https://github.com/googleapis/nodejs-pubsub/blob/main/samples/createTopicWithCloudStorageIngestion.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-pubsub&page=editor&open_in_editor=samples/createTopicWithCloudStorageIngestion.js,samples/README.md) + +__Usage:__ + + +`node createTopicWithCloudStorageIngestion.js ` + + +----- + + + + ### Create Topic With Kinesis Ingestion Creates a new topic, with Kinesis ingestion enabled. diff --git a/samples/createTopicWithCloudStorageIngestion.js b/samples/createTopicWithCloudStorageIngestion.js new file mode 100644 index 000000000..c66beeb6d --- /dev/null +++ b/samples/createTopicWithCloudStorageIngestion.js @@ -0,0 +1,118 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is a generated sample, using the typeless sample bot. Please +// look for the source TypeScript sample (.ts) for modifications. +'use strict'; + +/** + * This sample demonstrates how to perform basic operations on topics with + * the Google Cloud Pub/Sub API. + * + * For more information, see the README.md under /pubsub and the documentation + * at https://cloud.google.com/pubsub/docs. + */ + +// sample-metadata: +// title: Create Topic With Cloud Storage Ingestion +// description: Creates a new topic, with Cloud Storage ingestion enabled. +// usage: node createTopicWithCloudStorageIngestion.js + +// [START pubsub_create_topic_with_cloud_storage_ingestion] +/** + * TODO(developer): Uncomment these variables before running the sample. + */ +// const topicNameOrId = 'YOUR_TOPIC_NAME_OR_ID'; +// const bucket = 'YOUR_BUCKET_NAME'; +// const inputFormat = 'text'; +// const textDelimiter = '\n'; +// const matchGlob = '**.txt'; +// const minimumObjectCreateTime = 'YYYY-MM-DDThh:mm:ssZ; + +// Imports the Google Cloud client library +const {PubSub} = require('@google-cloud/pubsub'); + +// Creates a client; cache this for further use +const pubSubClient = new PubSub(); + +async function createTopicWithCloudStorageIngestion( + topicNameOrId, + bucket, + inputFormat, + textDelimiter, + matchGlob, + minimumObjectCreateTime +) { + const minimumDate = Date.parse(minimumObjectCreateTime); + const topicMetadata = { + name: topicNameOrId, + ingestionDataSourceSettings: { + cloudStorage: { + bucket, + minimumObjectCreateTime: { + seconds: minimumDate / 1000, + nanos: (minimumDate % 1000) * 1000, + }, + matchGlob, + }, + }, + }; + + // Make a format appropriately. + switch (inputFormat) { + case 'text': + topicMetadata.ingestionDataSourceSettings.cloudStorage.textFormat = { + delimiter: textDelimiter, + }; + break; + case 'avro': + topicMetadata.ingestionDataSourceSettings.cloudStorage.avroFormat = {}; + break; + case 'pubsub_avro': + topicMetadata.ingestionDataSourceSettings.cloudStorage.pubsubAvroFormat = + {}; + break; + default: + console.error('inputFormat must be in ("text", "avro", "pubsub_avro")'); + return; + } + + // Creates a new topic with Cloud Storage ingestion. + await pubSubClient.createTopic(topicMetadata); + console.log(`Topic ${topicNameOrId} created with Cloud Storage ingestion.`); +} +// [END pubsub_create_topic_with_cloud_storage_ingestion] + +function main( + topicNameOrId = 'YOUR_TOPIC_NAME_OR_ID', + bucket = 'YOUR_BUCKET_NAME', + inputFormat = 'text', + textDelimiter = '\n', + matchGlob = '**.txt', + minimumObjectCreateTime = 'YYYY-MM-DDThh:mm:ssZ' +) { + createTopicWithCloudStorageIngestion( + topicNameOrId, + bucket, + inputFormat, + textDelimiter, + matchGlob, + minimumObjectCreateTime + ).catch(err => { + console.error(err.message); + process.exitCode = 1; + }); +} + +main(...process.argv.slice(2)); diff --git a/samples/createTopicWithKinesisIngestion.js b/samples/createTopicWithKinesisIngestion.js index 3a77dcfa6..d9a50322e 100644 --- a/samples/createTopicWithKinesisIngestion.js +++ b/samples/createTopicWithKinesisIngestion.js @@ -52,8 +52,7 @@ async function createTopicWithKinesisIngestion( streamArn, consumerArn ) { - // Creates a new topic with a schema. Note that you might also - // pass Encodings.Json or Encodings.Binary here. + // Creates a new topic with Kinesis ingestion. await pubSubClient.createTopic({ name: topicNameOrId, ingestionDataSourceSettings: { diff --git a/samples/package.json b/samples/package.json index aff99d498..083fd4e4a 100644 --- a/samples/package.json +++ b/samples/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.0.0", - "@google-cloud/pubsub": "^4.7.2", + "@google-cloud/pubsub": "^4.8.0", "@google-cloud/storage": "^7.11.1", "@opentelemetry/api": "^1.6.0", "@opentelemetry/resources": "^1.17.0", diff --git a/samples/system-test/topics.test.ts b/samples/system-test/topics.test.ts index de31c71f3..54c28ee7f 100644 --- a/samples/system-test/topics.test.ts +++ b/samples/system-test/topics.test.ts @@ -13,6 +13,7 @@ // limitations under the License. import {Message, PubSub, Topic, Subscription} from '@google-cloud/pubsub'; +import {Bucket, Storage} from '@google-cloud/storage'; import {assert} from 'chai'; import {describe, it, after} from 'mocha'; import {execSync, commandFor} from './common'; @@ -52,6 +53,17 @@ describe('topics', () => { return {t: topic, tname, s: sub}; } + async function createStorageBucket(testName: string): Promise { + const storage = new Storage({ + projectId, + }); + + const name = resources.generateStorageName(testName); + + const [bucket] = await storage.createBucket(name); + return bucket; + } + async function cleanSubs() { const [subscriptions] = await pubsub.getSubscriptions(); await Promise.all( @@ -121,6 +133,27 @@ describe('topics', () => { assert.ok(exists, 'Topic was created'); }); + it('should create a topic with cloud storage ingestion', async () => { + const testId = 'create-gcs-ingestion'; + const name = topicName(testId); + const bucket = await createStorageBucket(testId); + const bucketName = bucket.name; + + try { + const output = execSync( + `${commandFor('createTopicWithCloudStorageIngestion')} ${name} ${ + bucketName + } text '\n' '**.txt' '2024-10-10T00:00:00Z'` + ); + assert.include(output, `Topic ${name} created with Cloud Storage ingestion.`); + const [topics] = await pubsub.getTopics(); + const exists = topics.some(t => t.name === fullTopicName(name)); + assert.ok(exists, 'Topic was created'); + } finally { + await bucket.delete(); + } + }); + it('should update a topic with kinesis integration', async () => { const pair = await createPair('update-kinesis'); const output = execSync( diff --git a/samples/typescript/createTopicWithCloudStorageIngestion.ts b/samples/typescript/createTopicWithCloudStorageIngestion.ts new file mode 100644 index 000000000..c30035bba --- /dev/null +++ b/samples/typescript/createTopicWithCloudStorageIngestion.ts @@ -0,0 +1,114 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * This sample demonstrates how to perform basic operations on topics with + * the Google Cloud Pub/Sub API. + * + * For more information, see the README.md under /pubsub and the documentation + * at https://cloud.google.com/pubsub/docs. + */ + +// sample-metadata: +// title: Create Topic With Cloud Storage Ingestion +// description: Creates a new topic, with Cloud Storage ingestion enabled. +// usage: node createTopicWithCloudStorageIngestion.js + +// [START pubsub_create_topic_with_cloud_storage_ingestion] +/** + * TODO(developer): Uncomment these variables before running the sample. + */ +// const topicNameOrId = 'YOUR_TOPIC_NAME_OR_ID'; +// const bucket = 'YOUR_BUCKET_NAME'; +// const inputFormat = 'text'; +// const textDelimiter = '\n'; +// const matchGlob = '**.txt'; +// const minimumObjectCreateTime = 'YYYY-MM-DDThh:mm:ssZ; + +// Imports the Google Cloud client library +import {PubSub, TopicMetadata} from '@google-cloud/pubsub'; + +// Creates a client; cache this for further use +const pubSubClient = new PubSub(); + +async function createTopicWithCloudStorageIngestion( + topicNameOrId: string, + bucket: string, + inputFormat: string, + textDelimiter: string, + matchGlob: string, + minimumObjectCreateTime: string +) { + const minimumDate = Date.parse(minimumObjectCreateTime); + const topicMetadata: TopicMetadata = { + name: topicNameOrId, + ingestionDataSourceSettings: { + cloudStorage: { + bucket, + minimumObjectCreateTime: { + seconds: minimumDate / 1000, + nanos: (minimumDate % 1000) * 1000, + }, + matchGlob, + }, + }, + }; + + // Make a format appropriately. + switch (inputFormat) { + case 'text': + topicMetadata.ingestionDataSourceSettings!.cloudStorage!.textFormat = { + delimiter: textDelimiter, + }; + break; + case 'avro': + topicMetadata.ingestionDataSourceSettings!.cloudStorage!.avroFormat = {}; + break; + case 'pubsub_avro': + topicMetadata.ingestionDataSourceSettings!.cloudStorage!.pubsubAvroFormat = + {}; + break; + default: + console.error('inputFormat must be in ("text", "avro", "pubsub_avro")'); + return; + } + + // Creates a new topic with Cloud Storage ingestion. + await pubSubClient.createTopic(topicMetadata); + console.log(`Topic ${topicNameOrId} created with Cloud Storage ingestion.`); +} +// [END pubsub_create_topic_with_cloud_storage_ingestion] + +function main( + topicNameOrId = 'YOUR_TOPIC_NAME_OR_ID', + bucket = 'YOUR_BUCKET_NAME', + inputFormat = 'text', + textDelimiter = '\n', + matchGlob = '**.txt', + minimumObjectCreateTime = 'YYYY-MM-DDThh:mm:ssZ' +) { + createTopicWithCloudStorageIngestion( + topicNameOrId, + bucket, + inputFormat, + textDelimiter, + matchGlob, + minimumObjectCreateTime + ).catch(err => { + console.error(err.message); + process.exitCode = 1; + }); +} + +main(...process.argv.slice(2)); diff --git a/samples/typescript/createTopicWithKinesisIngestion.ts b/samples/typescript/createTopicWithKinesisIngestion.ts index 7c225b227..6b42d4dc1 100644 --- a/samples/typescript/createTopicWithKinesisIngestion.ts +++ b/samples/typescript/createTopicWithKinesisIngestion.ts @@ -48,8 +48,7 @@ async function createTopicWithKinesisIngestion( streamArn: string, consumerArn: string ) { - // Creates a new topic with a schema. Note that you might also - // pass Encodings.Json or Encodings.Binary here. + // Creates a new topic with Kinesis ingestion. await pubSubClient.createTopic({ name: topicNameOrId, ingestionDataSourceSettings: {