Skip to content
This repository has been archived by the owner on Apr 18, 2024. It is now read-only.

Commit

Permalink
Updated locks
Browse files Browse the repository at this point in the history
  • Loading branch information
SBurwash committed Mar 7, 2024
1 parent f136460 commit 75e6007
Show file tree
Hide file tree
Showing 2 changed files with 145 additions and 0 deletions.
111 changes: 111 additions & 0 deletions plugins/loaders/target-bigquery--adswerve.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
{
"plugin_type": "loaders",
"name": "target-bigquery",
"namespace": "target_bigquery",
"variant": "adswerve",
"label": "Google BigQuery",
"docs": "https://hub.meltano.com/loaders/target-bigquery--adswerve",
"repo": "https://github.com/adswerve/target-bigquery",
"pip_url": "git+https://github.com/adswerve/target-bigquery.git@0.12.0",
"description": "BigQuery loader",
"logo_url": "https://hub.meltano.com/assets/logos/loaders/bigquery.png",
"settings_group_validation": [
[
"project_id",
"dataset_id",
"location",
"credentials_path"
]
],
"settings": [
{
"name": "project_id",
"label": "Project ID",
"description": "BigQuery project"
},
{
"name": "dataset_id",
"value": "$MELTANO_EXTRACT__LOAD_SCHEMA",
"label": "Dataset ID",
"description": "BigQuery dataset.\n\nThe default value [will expand to](https://docs.meltano.com/guide/configuration.html#expansion-in-setting-values) the value of the [`load_schema` extra](https://docs.meltano.com/concepts/plugins#load-schema-extra) for the extractor used in the pipeline, which defaults to the extractor's namespace, e.g. `tap_gitlab` for [`tap-gitlab`](/extractors/gitlab).\n"
},
{
"name": "location",
"value": "us-central1",
"label": "Region",
"description": "Dataset Location. See https://cloud.google.com/bigquery/docs/locations.",
"placeholder": "us-central1"
},
{
"name": "credentials_path",
"env": "GOOGLE_APPLICATION_CREDENTIALS",
"value": "$MELTANO_PROJECT_ROOT/client_secrets.json",
"label": "Credentials Path",
"description": "Fully qualified path to `client_secrets.json` for your service account.\n\nSee the [\"Activate the Google BigQuery API\" section of the repository's README](https://github.com/adswerve/target-bigquery#step-1-activate-the-google-bigquery-api) and <https://cloud.google.com/docs/authentication/production>.\n\nBy default, this file is expected to be at the root of your project directory.\n"
},
{
"name": "validate_records",
"kind": "boolean",
"value": false,
"label": "Validate Records"
},
{
"name": "add_metadata_columns",
"kind": "boolean",
"value": false,
"label": "Add Metadata Columns",
"description": "Add `_time_extracted` and `_time_loaded` metadata columns"
},
{
"name": "replication_method",
"kind": "options",
"value": "append",
"label": "Replication Method",
"description": "The Replication method can be:\n\n`append`: Adding new rows to the table (Default value)\n`truncate`: Deleting all previous rows and uploading the new ones to the table\n`incremental`: Upserting new rows into the table, using the primary key given by the tap connector (if it finds an old row with same key, updates it. Otherwise it inserts the new row)\n\nWARNING: It is not recommended to use the `incremental` option (which uses MERGE SQL statement). It might result in loss of production data, because historical records get updated. Instead, we recommend using the `append` replication method, which will preserve historical data.\n",
"options": [
{
"label": "Append",
"value": "append"
},
{
"label": "Truncate",
"value": "truncate"
},
{
"label": "Incremental",
"value": "incremental"
}
]
},
{
"name": "table_prefix",
"label": "Table Prefix",
"description": "Add prefix to table name"
},
{
"name": "table_suffix",
"label": "Table Suffix",
"description": "Add suffix to table name"
},
{
"name": "max_cache",
"value": 50,
"label": "Max Cache",
"description": "Maximum cache size in MB"
},
{
"name": "merge_state_messages",
"kind": "boolean",
"value": false,
"label": "Merge State Messages",
"description": "Whether to merge multiple state messages from the tap into the state file or uses the last state message as the state file. Note that it is not recommended to set this to true when using with Meltano as the merge behavior conflicts with Meltano\u2019s merge process."
},
{
"name": "table_config",
"label": "Table Config",
"description": "A path to a file containing the definition of partitioning and clustering."
}
],
"dialect": "bigquery",
"target_schema": "$TARGET_BIGQUERY_DATASET_ID"
}
34 changes: 34 additions & 0 deletions plugins/loaders/target-jsonl--andyh1203.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"plugin_type": "loaders",
"name": "target-jsonl",
"namespace": "target_jsonl",
"variant": "andyh1203",
"label": "JSON Lines (JSONL)",
"docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203",
"repo": "https://github.com/andyh1203/target-jsonl",
"pip_url": "target-jsonl",
"description": "JSONL loader",
"logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png",
"settings": [
{
"name": "destination_path",
"kind": "string",
"value": "output",
"label": "Destination Path",
"description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n"
},
{
"name": "do_timestamp_file",
"kind": "boolean",
"value": false,
"label": "Include Timestamp in File Names",
"description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n"
},
{
"name": "custom_name",
"kind": "string",
"label": "Custom File Name Override",
"description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n"
}
]
}

0 comments on commit 75e6007

Please sign in to comment.