From 1d42ddc6dd6925bf422c85cc732c8ad8cb44ffa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Ko=C5=82da?= Date: Thu, 7 Dec 2017 20:35:09 +0100 Subject: [PATCH] views now deduplicate data from 2 days instead of 3 --- bq_schemas/bigquery_views/partition_metadata_v1_0.sql | 6 +++--- bq_schemas/bigquery_views/table_metadata_v1_0.sql | 6 +++--- .../bigquery_views_legacy_sql/partition_metadata_v1_0.sql | 6 +++--- .../bigquery_views_legacy_sql/table_metadata_v0_1.sql | 6 +++--- .../bigquery_views_legacy_sql/table_metadata_v1_0.sql | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bq_schemas/bigquery_views/partition_metadata_v1_0.sql b/bq_schemas/bigquery_views/partition_metadata_v1_0.sql index 784eb8d..2a2ee14 100644 --- a/bq_schemas/bigquery_views/partition_metadata_v1_0.sql +++ b/bq_schemas/bigquery_views/partition_metadata_v1_0.sql @@ -1,11 +1,11 @@ #standardSQL --- This view aggregates all partition metadata from last 3 days and deduplicates it based on partition reference. --- Deleted table/partition can be returned by this query up to 3 days. +-- This view aggregates all partition metadata from last 2 days and deduplicates it based on partition reference. +-- Deleted table/partition can be returned by this query up to 2 days. SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM ( SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type, row_number() OVER (PARTITION BY projectId, datasetId, tableId, partitionId ORDER BY snapshotTime DESC) AS rownum FROM `$PROJECT_ID.bigquery.partition_metadata_v1_0` WHERE -_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -2 DAY)) +_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -1 DAY)) AND CURRENT_TIMESTAMP()) WHERE rownum = 1 diff --git a/bq_schemas/bigquery_views/table_metadata_v1_0.sql b/bq_schemas/bigquery_views/table_metadata_v1_0.sql index 9a1e2d4..17876c2 100644 --- a/bq_schemas/bigquery_views/table_metadata_v1_0.sql +++ b/bq_schemas/bigquery_views/table_metadata_v1_0.sql @@ -1,11 +1,11 @@ #standardSQL --- This view aggregates all table metadata from last 3 days and deduplicates it based on table reference. --- Deleted table can be returned by this query up to 3 days. +-- This view aggregates all table metadata from last 2 days and deduplicates it based on table reference. +-- Deleted table can be returned by this query up to 2 days. SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM ( SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type, row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum FROM `$PROJECT_ID.bigquery.table_metadata_v1_0` WHERE -_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -2 DAY)) +_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -1 DAY)) AND CURRENT_TIMESTAMP()) WHERE rownum = 1 \ No newline at end of file diff --git a/bq_schemas/bigquery_views_legacy_sql/partition_metadata_v1_0.sql b/bq_schemas/bigquery_views_legacy_sql/partition_metadata_v1_0.sql index 5734a00..0af6022 100644 --- a/bq_schemas/bigquery_views_legacy_sql/partition_metadata_v1_0.sql +++ b/bq_schemas/bigquery_views_legacy_sql/partition_metadata_v1_0.sql @@ -1,10 +1,10 @@ --- This view aggregates all partition metadata from last 3 days and deduplicates it based on partition reference. --- Deleted table/partition can be returned by this query up to 3 days. +-- This view aggregates all partition metadata from last 2 days and deduplicates it based on partition reference. +-- Deleted table/partition can be returned by this query up to 2 days. SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM ( SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type, row_number() OVER (PARTITION BY projectId, datasetId, tableId, partitionId ORDER BY snapshotTime DESC) AS rownum, FROM [bigquery.partition_metadata_v1_0] WHERE -_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000)) +_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000)) AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP()))) WHERE rownum = 1 \ No newline at end of file diff --git a/bq_schemas/bigquery_views_legacy_sql/table_metadata_v0_1.sql b/bq_schemas/bigquery_views_legacy_sql/table_metadata_v0_1.sql index 17b7eb1..1d082ac 100644 --- a/bq_schemas/bigquery_views_legacy_sql/table_metadata_v0_1.sql +++ b/bq_schemas/bigquery_views_legacy_sql/table_metadata_v0_1.sql @@ -1,11 +1,11 @@ --- This view aggregates all data from last 3 days and deduplicates it based on table reference. --- Deleted table can be returned by this query up to 3 days. +-- This view aggregates all data from last 2 days and deduplicates it based on table reference. +-- Deleted table can be returned by this query up to 2 days. SELECT * FROM ( SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type, timePartitioning.type AS partitioningType, row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum, COUNT(partition.partitionId) WITHIN RECORD AS partitionCount FROM [bigquery.table_metadata_v0_1] WHERE -_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000)) +_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000)) AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP()))) WHERE rownum = 1 \ No newline at end of file diff --git a/bq_schemas/bigquery_views_legacy_sql/table_metadata_v1_0.sql b/bq_schemas/bigquery_views_legacy_sql/table_metadata_v1_0.sql index 30b6726..a1efcff 100644 --- a/bq_schemas/bigquery_views_legacy_sql/table_metadata_v1_0.sql +++ b/bq_schemas/bigquery_views_legacy_sql/table_metadata_v1_0.sql @@ -1,10 +1,10 @@ --- This view aggregates all table metadata from last 3 days and deduplicates it based on table reference. --- Deleted table can be returned by this query up to 3 days. +-- This view aggregates all table metadata from last 2 days and deduplicates it based on table reference. +-- Deleted table can be returned by this query up to 2 days. SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM ( SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type, row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum, FROM [bigquery.table_metadata_v1_0] WHERE -_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000)) +_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000)) AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP()))) WHERE rownum = 1 \ No newline at end of file