Add in generator for all fields in account postgres example

pflooky · Aug 10, 2023 · 7eaf189 · 7eaf189
1 parent 0945dd3
commit 7eaf189
Show file tree

Hide file tree

Showing 10 changed files with 125 additions and 3,800 deletions.
diff --git a/docs/get-started/docker.md b/docs/get-started/docker.md
@@ -27,7 +27,9 @@ Can set it to one of the following:
 If you want to test it out with your own setup, you can alter the corresponding files under [docs/sample/docker/data](https://github.com/pflooky/data-caterer-docs/tree/main/docs/sample/docker/data)
 
 
-### Run with multiple data sources (Postgres and CSV File)
+### Run with multiple data sources
+
+#### Postgres and CSV File
 
 ```shell
 PLAN=plan/scenario-based DATA_SOURCE=postgres docker-compose up -d datacaterer
@@ -39,6 +41,10 @@ docker exec docker-postgres-1 psql -Upostgres -d customer -c "SELECT * FROM acco
 You should be able to see the linked data between Postgres and the CSV file created along with 1 to 10 records per
 account_id, name combination in the CSV file.
 
+#### Postgres and JMS
+
+
+
 ### Run with custom data sources
 
 1. Create/alter plan under [`data/custom/plan`](https://github.com/pflooky/data-caterer-docs/tree/main/docs/sample/docker/data/custom/plan)

diff --git a/docs/sample/docker/data/custom/task/jdbc/postgres/postgres-account-task.yaml b/docs/sample/docker/data/custom/task/jdbc/postgres/postgres-account-task.yaml
@@ -9,34 +9,48 @@ steps:
     schema:
       fields:
         - name: "account_number"
-          type: "string"
           generator:
             type: "regex"
             options:
               regex: "ACC1[0-9]{5,10}"
         - name: "account_status"
-          type: "string"
           generator:
             type: "oneOf"
             options:
               oneOf:
                 - "open"
                 - "closed"
-        - name: "open_date"
-          type: "date"
-          generator:
-            type: "random"
         - name: "created_by"
-          type: "string"
           generator:
-            type: "random"
             options:
-              expression: "#{Name.username} #{Name.lastname}"
+              expression: "#{Name.username}"
+        - name: "created_by_fixed_length"
+          generator:
+            type: "sql"
+            options:
+              sql: "SUBSTR(created_by, 0, 5)"
         - name: "customer_id_int"
           type: "int"
           generator:
-            type: "random"
             options:
               isUnique: true
               min: 1
               max: 999999999
+        - name: "customer_id_smallint"
+          type: "short"
+        - name: "customer_id_bigint"
+          type: "long"
+        - name: "customer_id_decimal"
+          type: "decimal(26,3)"
+        - name: "customer_id_real"
+          type: "float"
+        - name: "customer_id_double"
+          type: "double"
+        - name: "open_date"
+          type: "date"
+        - name: "open_timestamp"
+          type: "timestamp"
+        - name: "last_opened_time"
+          type: "timestamp"
+        - name: "payload_bytes"
+          type: "binary"
diff --git a/docs/sample/docker/docker-compose.yaml b/docs/sample/docker/docker-compose.yaml
@@ -1,4 +1,4 @@
-version: "3.5"
+version: "3.9"
 services:
   datacaterer:
 #    image: "datacatering/data-caterer-basic:0.2.0"
@@ -24,12 +24,16 @@ services:
     volumes:
       - "${HOME}/data/postgres:/data/postgres"
       - "./data/sql/postgres/customer.sql:/docker-entrypoint-initdb.d/customer.sql"
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
     ports:
       - "5432:5432"
 
   postgres:
-    image: "postgres:14.5"
-    entrypoint: [ "exit" ]
+    image: "alpine:3.18.3"
     depends_on:
       postgresserver:
         condition: "service_healthy"
@@ -61,6 +65,10 @@ services:
     volumes:
       - "${HOME}/data/mysql:/var/lib/mysql"
       - "./data/sql/mysql/customer.sql:/docker-entrypoint-initdb.d/customer.sql"
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost", "-u", "root", "-p$$MYSQL_ROOT_PASSWORD" ]
+      timeout: 20s
+      retries: 10
     ports:
       - "3306:3306"
 

diff --git a/site/get-started/docker/index.html b/site/get-started/docker/index.html
@@ -341,10 +341,30 @@
 </li>
 
           <li class="md-nav__item">
-  <a href="#run-with-multiple-data-sources-postgres-and-csv-file" class="md-nav__link">
-    Run with multiple data sources (Postgres and CSV File)
+  <a href="#run-with-multiple-data-sources" class="md-nav__link">
+    Run with multiple data sources
   </a>
 
+    <nav class="md-nav" aria-label="Run with multiple data sources">
+      <ul class="md-nav__list">
+
+          <li class="md-nav__item">
+  <a href="#postgres-and-csv-file" class="md-nav__link">
+    Postgres and CSV File
+  </a>
+
+</li>
+
+          <li class="md-nav__item">
+  <a href="#postgres-and-jms" class="md-nav__link">
+    Postgres and JMS
+  </a>
+
+</li>
+
+      </ul>
+    </nav>
+
 </li>
 
           <li class="md-nav__item">
@@ -551,10 +571,30 @@
 </li>
 
           <li class="md-nav__item">
-  <a href="#run-with-multiple-data-sources-postgres-and-csv-file" class="md-nav__link">
-    Run with multiple data sources (Postgres and CSV File)
+  <a href="#run-with-multiple-data-sources" class="md-nav__link">
+    Run with multiple data sources
   </a>
 
+    <nav class="md-nav" aria-label="Run with multiple data sources">
+      <ul class="md-nav__list">
+
+          <li class="md-nav__item">
+  <a href="#postgres-and-csv-file" class="md-nav__link">
+    Postgres and CSV File
+  </a>
+
+</li>
+
+          <li class="md-nav__item">
+  <a href="#postgres-and-jms" class="md-nav__link">
+    Postgres and JMS
+  </a>
+
+</li>
+
+      </ul>
+    </nav>
+
 </li>
 
           <li class="md-nav__item">
@@ -638,14 +678,16 @@ <h3 id="quick-start">Quick start</h3>
 <li>http</li>
 </ul>
 <p>If you want to test it out with your own setup, you can alter the corresponding files under <a href="https://github.com/pflooky/data-caterer-docs/tree/main/docs/sample/docker/data">docs/sample/docker/data</a></p>
-<h3 id="run-with-multiple-data-sources-postgres-and-csv-file">Run with multiple data sources (Postgres and CSV File)</h3>
+<h3 id="run-with-multiple-data-sources">Run with multiple data sources</h3>
+<h4 id="postgres-and-csv-file">Postgres and CSV File</h4>
 <pre><code class="language-shell">PLAN=plan/scenario-based DATA_SOURCE=postgres docker-compose up -d datacaterer
 head data/custom/csv/transactions/part-00000*
 sample_account=$(head -1 data/custom/csv/transactions/part-00000* | awk -F &quot;,&quot; '{print $1}')
 docker exec docker-postgres-1 psql -Upostgres -d customer -c &quot;SELECT * FROM account.accounts WHERE account_number='$sample_account'&quot;
 </code></pre>
 <p>You should be able to see the linked data between Postgres and the CSV file created along with 1 to 10 records per
 account_id, name combination in the CSV file.</p>
+<h4 id="postgres-and-jms">Postgres and JMS</h4>
 <h3 id="run-with-custom-data-sources">Run with custom data sources</h3>
 <ol>
 <li>Create/alter plan under <a href="https://github.com/pflooky/data-caterer-docs/tree/main/docs/sample/docker/data/custom/plan"><code>data/custom/plan</code></a></li>

diff --git a/site/sample/docker/data/custom/generated/plan/plan_20230810T125502+0800.yaml b/site/sample/docker/data/custom/generated/plan/plan_20230810T125502+0800.yaml