Skip to content

Commit

Permalink
fix: use backslash
Browse files Browse the repository at this point in the history
  • Loading branch information
Ian2012 committed Aug 9, 2024
1 parent a288c03 commit 3972313
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 12 deletions.
7 changes: 3 additions & 4 deletions tutoraspects/patches/local-docker-compose-jobs-services
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@ aspects-job:
volumes:
- ../../env/plugins/aspects/apps/aspects:/app/aspects
- ../../env/plugins/aspects/apps/aspects/scripts/:/app/aspects/scripts:ro
{% if RUN_SUPERSET or RUN_CLICKHOUSE or RUN_RALPH %}depends_on:{% if RUN_SUPERSET %}
- superset{% endif %}{% if RUN_CLICKHOUSE%}
- clickhouse{% endif %}{% if RUN_RALPH %}
- ralph{% endif %}
{% if RUN_CLICKHOUSE %}
depends_on:
- clickhouse
{% endif %}
clickhouse-job:
image: {{DOCKER_IMAGE_CLICKHOUSE}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,5 @@ fi

echo "Installing dbt dependencies"
dbt deps

mkdir target
8 changes: 3 additions & 5 deletions tutoraspects/templates/aspects/apps/aspects/scripts/dbt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,17 @@ echo "Running ${@:2}"

python3 /app/aspects/scripts/insert_data.py load

dbt compile

# If state exists and we've asked to only run changed files, add the flag
if [ "$1" == "True" ] && [ -e "${DBT_STATE}manifest.json" ]
if [ "$1" == "True" ] && [ -e "${DBT_STATE}/manifest.json" ]
then
echo "Found ${DBT_STATE}manifest.json so only running modified items and their downstreams"
echo "Found ${DBT_STATE}/manifest.json so only running modified items and their downstreams"
${@:2} -s state:modified+
else
echo "Running command *without* state:modified+ this may take a long time."
${@:2}
fi

if [ -e "${DBT_STATE}manifest.json" ]
if [ -e "${DBT_STATE}/manifest.json" ]
then
echo "Updating dbt state..."
python3 /app/aspects/scripts/insert_data.py sink
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
DBT_PROJECT_ROOT = "/app/aspects-dbt"

DBT_STATE_DIR = "{{DBT_STATE_DIR}}"

print("Insert data script.")
client = clickhouse_connect.get_client(
host="{{CLICKHOUSE_HOST}}",
username='{{CLICKHOUSE_ADMIN_USER}}',
Expand All @@ -16,7 +16,7 @@

def sink_files():
files = []
file_name = f"{DBT_STATE_DIR}manifest.json"
file_name = f"{DBT_STATE_DIR}/manifest.json"

with open(file_name, "r") as file:
content = file.read()
Expand All @@ -37,9 +37,11 @@ def sink_files():

def load_files():
result = client.query("SELECT path, content from {{ ASPECTS_EVENT_SINK_DATABASE }}.aspects_data OPTIMIZE FINAL")
if not result.result_rows:
print("There is no state stored.")
for row in result.result_rows:
path, content = row
file_path = f"{DBT_STATE_DIR}manifest.json"
file_path = f"{DBT_STATE_DIR}/manifest.json"
with open(file_path, "w") as f:
print(f"Loading: {file_path}")
f.write(content)
Expand Down

0 comments on commit 3972313

Please sign in to comment.