Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Updated with common schema files and seperated postgres data migration #425

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Updated with common schema files and seperated postgres data migration #425

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
460d426
Updated with common schema files and seperated postgres data migration
chenchals Jun 2, 2023
6d12c40
set export TANAGRA_DB_INITIALIZE_ON_START=true
chenchals Jun 2, 2023
07eace6
Updated with common schema files and seperated postgres data migration
chenchals Jun 2, 2023
f8613d3
set export TANAGRA_DB_INITIALIZE_ON_START=true
chenchals Jun 2, 2023
fa1775c
Updated to use column names property_key, property_value, criteria_ke…
chenchals Jun 5, 2023
bc4b30e
Merge remote-tracking branch 'origin/chenchals/DT-409-Aou-Tanagra-com…
chenchals Jun 5, 2023
f950f3d
Changed TANAGRA_DB_INITIALIZE_ON_START=false as appropriate for main …
chenchals Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion service/local-dev/run_server.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ while getopts ":av" arg; do
done

export TANAGRA_DATABASE_NAME=tanagra_db
export TANAGRA_DB_INITIALIZE_ON_START=false
export TANAGRA_DB_INITIALIZE_ON_START=true
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is helpful for testing, but please leave it set to false in the main branch. This is the setting we'll deploy with, so I'd like to keep it as the testing default as well.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!

export TANAGRA_DB_USERNAME=dbuser
export TANAGRA_DB_PASSWORD=dbpwd

Expand Down
3 changes: 2 additions & 1 deletion service/src/main/resources/db/changelog.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@
<property global="true" dbms="mariadb,mysql" name="jsonb.type" value="longtext"/>
<!-- define changesets -->
<include file="changesets/20230410_schema_reset.yaml" relativeToChangelogFile="true"/>
<include file="changesets/20230530_avoid_postgres_specific_features.yaml" relativeToChangelogFile="true"/>
<include file="changesets/20230602_schema_extend.yaml" relativeToChangelogFile="true"/>
<include file="changesets/20230602_postgres_data_migrate.yaml" relativeToChangelogFile="true"/>
</databaseChangeLog>
22 changes: 22 additions & 0 deletions service/src/main/resources/db/changesets/20230602_postgres_data_migrate.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
databaseChangeLog:
- changeSet:
id: postgres_data_migrate
author: marikomedlock, chenchalsubraveti
dbms: postgresql
# TODO: Remove the jsonb and 2 text[] columns in a follow-on changeset, once we're sure this migration was successful.
# study.properties, criteria.tags, annotation_key.enumVals
changes:
- sql:
# Migrate the properties of existing studies from the JSON column to this table.
sql: |
INSERT INTO study_property (study_id, key, value)
SELECT id, key, properties->>key as value
FROM (SELECT id, jsonb_object_keys(properties) as key, properties FROM study) AS study_property_keys;

# No migration SQL because the criteria.tags text[] column is not yet being used by the UI.

- sql:
# Migrate the enum values of existing annotation keys from the text[] column to this table.
sql: |
INSERT INTO annotation_key_enum_value (cohort_id, annotation_key_id, enum)
SELECT cohort_id, id AS annotation_key_id, unnest(enum_vals) AS enum FROM annotation_key;
34 changes: 12 additions & 22 deletions ...530_avoid_postgres_specific_features.yaml → ...db/changesets/20230602_schema_extend.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
databaseChangeLog:
- changeSet:
id: avoid_postgres_specific_features
id: schema_extend
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This changeset file has already been deployed to our test instance (we deploy nightly), so you can't change the name or any of the schema specification for postgres. If you change it to render differently for mariadb/mysql only, then I think that's okay (though I'd want to test to confirm) but changing the postgres rendering will definitely break things.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not follow.

author: marikomedlock, chenchalsubraveti
# TODO: Remove the jsonb and 2 text[] columns in a follow-on changeset, once we're sure this migration was successful.
# study.properties, criteria.tags, annotation_key.enumVals
changes:
- createTable:
tableName: study_property
Expand All @@ -13,7 +11,7 @@ databaseChangeLog:
type: ${id.type}
constraints:
references: study(id)
foreignKeyName: fk_sp_s
foreignKeyName: fk_sp_s_2
nullable: false
deleteCascade: true
remarks: Deleting a study will cascade to delete its properties
Expand All @@ -28,15 +26,9 @@ databaseChangeLog:
constraints:
nullable: true
- addUniqueConstraint:
constraintName: pk_sp
constraintName: pk_sp_2
tableName: study_property
columnNames: study_id, key
- sql:
# Migrate the properties of existing studies from the JSON column to this table.
sql: |
INSERT INTO study_property (study_id, key, value)
SELECT id, key, properties->>key as value
FROM (SELECT id, jsonb_object_keys(properties) as key, properties FROM study) AS study_property_keys;

- createTable:
tableName: criteria_tag
Expand All @@ -61,7 +53,7 @@ databaseChangeLog:
type: ${id.type}
constraints:
references: cohort_revision(id)
foreignKeyName: ck_crit_cr
foreignKeyName: ck_crit_cr_2
nullable: true
deleteCascade: true
remarks: Deleting a cohort revision will cascade to delete the criteria contained in it
Expand All @@ -70,7 +62,7 @@ databaseChangeLog:
type: ${id.type}
constraints:
references: concept_set(id)
foreignKeyName: fk_crit_cs
foreignKeyName: fk_crit_cs_2
nullable: true
deleteCascade: true
remarks: Deleting a concept set will cascade to delete the criteria contained in it
Expand All @@ -85,18 +77,19 @@ databaseChangeLog:
constraints:
nullable: false
- addUniqueConstraint:
constraintName: pk_ct
constraintName: pk_ct_2
tableName: criteria_tag
columnNames: criteria_id, criteria_group_id, criteria_group_section_id, cohort_revision_id, concept_set_id, key
- addForeignKeyConstraint:
constraintName: fk_ct_c
constraintName: fk_ct_c_2
baseColumnNames: criteria_id, criteria_group_id, criteria_group_section_id, cohort_revision_id, concept_set_id
baseTableName: criteria_tag
referencedColumnNames: id, criteria_group_id, criteria_group_section_id, cohort_revision_id, concept_set_id
referencedTableName: criteria
- dropNotNullConstraint:
tableName: criteria
columnName: tags
columnDataType: ${text.array.type}
# No migration SQL because the criteria.tags text[] column is not yet being used by the UI.

- createTable:
Expand All @@ -118,11 +111,11 @@ databaseChangeLog:
constraints:
nullable: false
- addUniqueConstraint:
constraintName: pk_akev
constraintName: pk_akev_2
tableName: annotation_key_enum_value
columnNames: enum, annotation_key_id, cohort_id
- addForeignKeyConstraint:
constraintName: fk_akev_ak
constraintName: fk_akev_ak_2
baseColumnNames: annotation_key_id, cohort_id
baseTableName: annotation_key_enum_value
referencedColumnNames: id, cohort_id
Expand All @@ -132,8 +125,5 @@ databaseChangeLog:
- dropNotNullConstraint:
tableName: annotation_key
columnName: enum_vals
- sql:
# Migrate the enum values of existing annotation keys from the text[] column to this table.
sql: |
INSERT INTO annotation_key_enum_value (cohort_id, annotation_key_id, enum)
SELECT cohort_id, id AS annotation_key_id, unnest(enum_vals) AS enum FROM annotation_key;
columnDataType: ${text.array.type}