Deploying to gh-pages from @ 9114025 🚀

rj_cor/meteorologia/precipitacao_alertario/schedules.html

@@ -43,7 +43,7 @@ <h1 class="title">Module <code>pipelines.rj_cor.meteorologia.precipitacao_alerta
 minute_schedule = Schedule(
     clocks=[
         IntervalClock(
-            interval=timedelta(minutes=5),
+            interval=timedelta(minutes=3),
             start_date=datetime(2021, 1, 1, 0, 1, 0),
             labels=[
                 constants.RJ_COR_AGENT_LABEL.value,

rj_cor/meteorologia/precipitacao_alertario/tasks.html

@@ -53,6 +53,7 @@ <h1 class="title">Module <code>pipelines.rj_cor.meteorologia.precipitacao_alerta
 from pipelines.utils.utils import (
     build_redis_key,
     compare_dates_between_tables_redis,
+    get_redis_output,
     log,
     to_partitions,
     save_str_on_redis,
@@ -230,9 +231,13 @@ <h1 class="title">Module <code>pipelines.rj_cor.meteorologia.precipitacao_alerta
     &#34;&#34;&#34;
     Save on dbt last timestamp where it was updated
     &#34;&#34;&#34;
-    now = pendulum.now(&#34;America/Sao_Paulo&#34;).to_datetime_string()
+    last_update_key = build_redis_key(
+        dataset_id, table_id, name=&#34;last_update&#34;, mode=mode
+    )
+    last_update = get_redis_output(last_update_key)
     redis_key = build_redis_key(dataset_id, table_id, name=&#34;dbt_last_update&#34;, mode=mode)
-    save_str_on_redis(redis_key, &#34;date&#34;, now)
+    log(f&#34;Saving {last_update} as last time dbt was updated&#34;)
+    save_str_on_redis(redis_key, &#34;date&#34;, last_update[&#34;date&#34;])
 
 
 @task(skip_on_upstream_skip=False)
@@ -361,9 +366,13 @@ <h2 class="section-title" id="header-functions">Functions</h2>
     &#34;&#34;&#34;
     Save on dbt last timestamp where it was updated
     &#34;&#34;&#34;
-    now = pendulum.now(&#34;America/Sao_Paulo&#34;).to_datetime_string()
+    last_update_key = build_redis_key(
+        dataset_id, table_id, name=&#34;last_update&#34;, mode=mode
+    )
+    last_update = get_redis_output(last_update_key)
     redis_key = build_redis_key(dataset_id, table_id, name=&#34;dbt_last_update&#34;, mode=mode)
-    save_str_on_redis(redis_key, &#34;date&#34;, now)</code></pre>
+    log(f&#34;Saving {last_update} as last time dbt was updated&#34;)
+    save_str_on_redis(redis_key, &#34;date&#34;, last_update[&#34;date&#34;])</code></pre>
 </details>
 </dd>
 <dt id="pipelines.rj_cor.meteorologia.precipitacao_alertario.tasks.tratar_dados"><code class="name flex">

utils/utils.html

@@ -991,6 +991,18 @@ <h1 class="title">Module <code>pipelines.utils.utils</code></h1>
     redis_client.hset(redis_key, key, value)
 
 
+def get_redis_output(redis_key):
+    &#34;&#34;&#34;
+    Get Redis output
+    Example: {b&#39;date&#39;: b&#39;2023-02-27 07:29:04&#39;}
+    &#34;&#34;&#34;
+    redis_client = get_redis_client()
+    output = redis_client.hgetall(redis_key)
+    if len(output) &gt; 0:
+        output = treat_redis_output(output)
+    return output
+
+
 def treat_redis_output(text):
     &#34;&#34;&#34;
     Redis returns a dict where both key and value are byte string
@@ -1010,23 +1022,20 @@ <h1 class="title">Module <code>pipelines.utils.utils</code></h1>
     table is bigger then the first one
     &#34;&#34;&#34;
 
-    redis_client = get_redis_client()
-
     # get saved date on redis
-    date_1 = redis_client.hgetall(key_table_1)
-    date_2 = redis_client.hgetall(key_table_2)
+    date_1 = get_redis_output(key_table_1)
+    date_2 = get_redis_output(key_table_2)
 
+    # Return true if there is no date_1 or date_2 saved on redis
     if (len(date_1) == 0) | (len(date_2) == 0):
         return True
 
-    date_1 = treat_redis_output(date_1)
-    date_2 = treat_redis_output(date_2)
-
     # Convert date to pendulum
     date_1 = pendulum.from_format(date_1[&#34;date&#34;], format_date_table_1)
     date_2 = pendulum.from_format(date_2[&#34;date&#34;], format_date_table_2)
-
-    return date_1 &lt; date_2
+    comparison = date_1 &lt; date_2
+    log(f&#34;Is {date_2} bigger than {date_1}? {comparison}&#34;)
+    return comparison
 
 
 # pylint: disable=W0106
@@ -1232,23 +1241,20 @@ <h2 class="section-title" id="header-functions">Functions</h2>
     table is bigger then the first one
     &#34;&#34;&#34;
 
-    redis_client = get_redis_client()
-
     # get saved date on redis
-    date_1 = redis_client.hgetall(key_table_1)
-    date_2 = redis_client.hgetall(key_table_2)
+    date_1 = get_redis_output(key_table_1)
+    date_2 = get_redis_output(key_table_2)
 
+    # Return true if there is no date_1 or date_2 saved on redis
     if (len(date_1) == 0) | (len(date_2) == 0):
         return True
 
-    date_1 = treat_redis_output(date_1)
-    date_2 = treat_redis_output(date_2)
-
     # Convert date to pendulum
     date_1 = pendulum.from_format(date_1[&#34;date&#34;], format_date_table_1)
     date_2 = pendulum.from_format(date_2[&#34;date&#34;], format_date_table_2)
-
-    return date_1 &lt; date_2</code></pre>
+    comparison = date_1 &lt; date_2
+    log(f&#34;Is {date_2} bigger than {date_1}? {comparison}&#34;)
+    return comparison</code></pre>
 </details>
 </dd>
 <dt id="pipelines.utils.utils.dataframe_to_csv"><code class="name flex">
@@ -1523,6 +1529,28 @@ <h2 id="returns">Returns</h2>
     )</code></pre>
 </details>
 </dd>
+<dt id="pipelines.utils.utils.get_redis_output"><code class="name flex">
+<span>def <span class="ident">get_redis_output</span></span>(<span>redis_key)</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Get Redis output
+Example: {b'date': b'2023-02-27 07:29:04'}</p></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def get_redis_output(redis_key):
+    &#34;&#34;&#34;
+    Get Redis output
+    Example: {b&#39;date&#39;: b&#39;2023-02-27 07:29:04&#39;}
+    &#34;&#34;&#34;
+    redis_client = get_redis_client()
+    output = redis_client.hgetall(redis_key)
+    if len(output) &gt; 0:
+        output = treat_redis_output(output)
+    return output</code></pre>
+</details>
+</dd>
 <dt id="pipelines.utils.utils.get_storage_blobs"><code class="name flex">
 <span>def <span class="ident">get_storage_blobs</span></span>(<span>dataset_id: str, table_id: str, mode: str = 'staging') ‑> list</span>
 </code></dt>
@@ -2679,6 +2707,7 @@ <h1>Index</h1>
 <li><code><a title="pipelines.utils.utils.final_column_treatment" href="#pipelines.utils.utils.final_column_treatment">final_column_treatment</a></code></li>
 <li><code><a title="pipelines.utils.utils.get_credentials_from_env" href="#pipelines.utils.utils.get_credentials_from_env">get_credentials_from_env</a></code></li>
 <li><code><a title="pipelines.utils.utils.get_redis_client" href="#pipelines.utils.utils.get_redis_client">get_redis_client</a></code></li>
+<li><code><a title="pipelines.utils.utils.get_redis_output" href="#pipelines.utils.utils.get_redis_output">get_redis_output</a></code></li>
 <li><code><a title="pipelines.utils.utils.get_storage_blobs" href="#pipelines.utils.utils.get_storage_blobs">get_storage_blobs</a></code></li>
 <li><code><a title="pipelines.utils.utils.get_username_and_password_from_secret" href="#pipelines.utils.utils.get_username_and_password_from_secret">get_username_and_password_from_secret</a></code></li>
 <li><code><a title="pipelines.utils.utils.get_vault_client" href="#pipelines.utils.utils.get_vault_client">get_vault_client</a></code></li>