Sqitch integration

docker-compose.yml

@@ -70,6 +70,14 @@ services:
       - POSTGRES_USER=${PG_USER}
       - POSTGRES_PASSWORD=${PG_PASSWORD}
 
+  sqitch:
+    container_name: sqitch
+    build:
+      context: ./sqitch
+      dockerfile: ./Dockerfile
+    depends_on:
+      - db
+
   staging_db:
     container_name: pg_container_testing
     image: postgres

sqitch/Dockerfile

@@ -0,0 +1 @@
+FROM sqitch/sqitch:latest

sqitch/README.md

@@ -0,0 +1,96 @@
+# Sqitch
+Sqitch is an open source database change management tool, basically like git but specifically for databases.
+
+First, Make sure will ran `make dev-build`, so that the sqitch container is running.
+Then cd into the `sqitch` folder.
+As a sanity check, type `./sqitch help` into the command line and check that the following pops up:
+```
+Usage
+      sqitch [--etc-path | --help | --man | --version]
+      sqitch <command> [--chdir <path>] [--no-pager] [--quiet] [--verbose]
+             [<command-options>] [<args>]
+
+Common Commands
+    The most commonly used sqitch commands are:
+
+      add        Add a new change to the plan
+      bundle     Bundle a Sqitch project for distribution
+      checkout   Revert, checkout another VCS branch, and re-deploy changes
+      config     Get and set local, user, or system options
+      deploy     Deploy changes to a database
+      engine     Manage database engine configuration
+      help       Display help information about Sqitch commands
+      init       Initialize a project
+      log        Show change logs for a database
+      plan       Show the contents of a plan
+      rebase     Revert and redeploy database changes
+      revert     Revert changes from a database
+      rework     Duplicate a change in the plan and revise its scripts
+      show       Show information about changes and tags, or change script contents
+      status     Show the current deployment status of a database
+      tag        Add or list tags in the plan
+      target     Manage target database configuration
+      upgrade    Upgrade the registry to the current version
+      verify     Verify changes to a database
+
+    See "sqitch help <command>" or "sqitch help <concept>" to read about a
+    specific command or concept. See "sqitch help --guide" for a list of
+    conceptual guides.
+```
+
+## User config
+Much like git, it will be great if we can tell who made a certain change to the database.
+To do that, simply run the following commands.
+`sqitch config --user user.name '<name>'`
+`sqitch config --user user.email '<email>'`
+
+## Adding a new SQL script to the database
+To add a new change to the database (Like a new SQl script, table or function), first run
+`./sqitch add appschema -n "<message>"`
+
+You will see the following changes:
+1. New SQL files will be made in the `/deploy`, `/revert` and `/verify` folders.
+2. The `sqitch.plan` file will be appended with your new change and message.
+
+The `/deploy` folder contains all the SQL scripts that will be ran when we deploy the database.
+The `/revert` folder contains all the SQL scripts that will be ran when we need to revert the database to some previous state. (Usually just contains a bunch of DROP TABLE expressions)
+The `/verify` folder contains all the SQL scripts to verify that a deploy did what it's supposed to.
+
+## Deploying changes
+To deploy changes you made to the database simply type in:
+`./sqitch deploy test`
+
+This will cause sqitch to run all the SQl scripts inside your `/deploy` folder in the order they are created.
+
+## Revert changes
+To revert changes you made to a database, type
+`./sqitch revert test --to @HEAD^`
+
+The `@HEAD` always points to *the last change deployed to the database*
+The `^` appended tells Sqitch to select the change prior to the last deployed change.
+You can add more `^` to go back further.
+You can also use `@ROOT` which refers to the first change deployed to the database.
+
+## Verify changes
+(Use this if you actually fill out the scripts inside of the `/verify` folder)
+
+Run `./sqitch verify test` to run verification checks.
+
+## Sqitch.conf
+This file contains all the settings that sqitch will follow, you don't need to touch anything here.
+The only thing of note is the 
+```
+[target "test"]
+	uri = db:pg://postgres:postgres@localhost:5432/test_db
+```
+
+This tells sqitch that `test` refers to the a specific database uri (Which is the current one we are using)
+So you don't have to type the actual uri itself if you want to use sqitch, you just type `test` instead.
+
+## The .bat and .sh files
+Don't remove them or touch them.
+The sqitch docker container need them in order for you to interact with sqitch from the command line.
+
+## References
+Refer to the official Sqitch page for more detailed documentation.
+https://sqitch.org/

sqitch/deploy/01-create_migration_table.sql

@@ -0,0 +1,18 @@
+-- Deploy website:01-create_migration_table to pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+CREATE TABLE IF NOT EXISTS migrations (
+   MigrationID SERIAL PRIMARY KEY,
+   VersionID INTEGER default 0
+);
+
+DO LANGUAGE plpgsql $$
+BEGIN
+   IF NOT EXISTS (SELECT FROM migrations WHERE MigrationID = 1) THEN
+      INSERT INTO migrations (MigrationID, VersionID) VALUES (1, 0);
+   END IF;
+END $$;
+
+COMMIT;

sqitch/deploy/02-create_frontend_table.sql

@@ -0,0 +1,12 @@
+-- Deploy website:02-create_frontend_table to pg
+-- requires: 01-create_migration_table
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS frontend (
+  FrontendID  SERIAL PRIMARY KEY,
+  FrontendURL VARCHAR(100)
+);
+-- XXX Add DDLs here.
+
+COMMIT;

sqitch/deploy/03-create_groups_table.sql

@@ -0,0 +1,19 @@
+-- Deploy website:03-create_groups_table to pg
+-- requires: 02-create_frontend_table
+
+BEGIN;
+
+-- XXX Add DDLs here.
+CREATE EXTENSION IF NOT EXISTS hstore;
+SET timezone = 'Australia/Sydney';
+
+DROP TYPE IF EXISTS permissions_enum CASCADE;
+CREATE TYPE permissions_enum as ENUM ('read', 'write', 'delete');
+
+CREATE TABLE IF NOT EXISTS groups (
+  UID   SERIAL PRIMARY KEY,
+  Name  VARCHAR(50) NOT NULL,
+  Permission permissions_enum UNIQUE NOT NULL
+);
+
+COMMIT;

sqitch/deploy/04-create_person_table.sql

@@ -0,0 +1,38 @@
+-- Deploy website:04-create_person_table to pg
+-- requires: 03-create_groups_table
+
+BEGIN;
+
+-- XXX Add DDLs here.
+CREATE TABLE IF NOT EXISTS person (
+  UID           SERIAL PRIMARY KEY,
+  Email         VARCHAR(50) UNIQUE NOT NULL,
+  First_name    VARCHAR(50) NOT NULL,
+  Password      CHAR(64) NOT NULL,
+
+  isOfGroup     INT,
+  frontendid    INT, 
+
+  CONSTRAINT fk_AccessLevel FOREIGN KEY (isOfGroup)
+    REFERENCES groups(UID),
+
+  CONSTRAINT fk_AccessFrontend FOREIGN KEY (frontendid)
+    REFERENCES frontend(FrontendID),
+
+  /* non duplicate email and password constraints */
+  CONSTRAINT no_dupes UNIQUE (Email, Password)
+);
+
+/* create user function plpgsql */
+CREATE OR REPLACE FUNCTION create_normal_user (email VARCHAR, name VARCHAR, password VARCHAR, frontendID INT) RETURNS void
+LANGUAGE plpgsql
+AS $$
+DECLARE
+BEGIN
+  INSERT INTO person (Email, First_name, Password, isOfGroup, frontendID)
+    VALUES (email, name, encode(sha256(password::BYTEA), 'hex'), 2, 1);
+END $$;
+
+
+
+COMMIT;

sqitch/deploy/05-create_filesystem_table.sql

@@ -0,0 +1,93 @@
+-- Deploy website:05-create_filesystem_table to pg
+-- requires: 04-create_person_table
+
+BEGIN;
+
+-- XXX Add DDLs here.
+SET timezone = 'Australia/Sydney';
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+/* MetaData */
+CREATE TABLE IF NOT EXISTS metadata (
+  MetadataID    uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+  CreatedAt     TIMESTAMP NOT NULL DEFAULT NOW()
+);
+
+/**
+  The filesystem table models all file heirachies in our system
+**/
+CREATE TABLE IF NOT EXISTS filesystem (
+  EntityID      uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+  LogicalName   VARCHAR(50) NOT NULL,
+
+  IsDocument    BOOLEAN DEFAULT false,
+  IsPublished   BOOLEAN DEFAULT false,
+  CreatedAt     TIMESTAMP NOT NULL DEFAULT NOW(),
+
+  /* MetaData */
+  -- MetadataID        uuid NOT NULL,
+
+  OwnedBy       INT,
+  /* Pain */
+  Parent        uuid REFERENCES filesystem(EntityID) DEFAULT NULL,
+
+  /* FK Constraint */
+  CONSTRAINT fk_owner FOREIGN KEY (OwnedBy) 
+    REFERENCES groups(UID),
+
+  -- CONSTRAINT fk_meta FOREIGN KEY (MetadataID) REFERENCES metadata(MetadataID),
+
+  /* Unique name constraint: there should not exist an entity of the same type with the
+     same parent and logical name. */
+  CONSTRAINT unique_name UNIQUE (Parent, LogicalName, IsDocument)        
+);
+
+/* Utility procedure :) */
+CREATE OR REPLACE FUNCTION new_entity (parentP uuid, logicalNameP VARCHAR, ownedByP INT, isDocumentP BOOLEAN DEFAULT false) RETURNS uuid
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  newEntityID filesystem.EntityID%type;
+  parentIsDocument BOOLEAN := (SELECT IsDocument FROM filesystem WHERE EntityID = parentP LIMIT 1);
+BEGIN
+  IF parentIsDocument THEN
+    /* We shouldnt be delcaring that a document is our parent */
+    RAISE EXCEPTION SQLSTATE '90001' USING MESSAGE = 'cannot make parent a document';
+  END If;
+  WITH newEntity AS (
+    INSERT INTO filesystem (LogicalName, IsDocument, OwnedBy, Parent)
+      VALUES (logicalNameP, isDocumentP, ownedByP, parentP)
+      RETURNING EntityID
+  )
+
+  SELECT newEntity.EntityID INTO newEntityID FROM newEntity;
+  RETURN newEntityID;
+END $$;
+
+/* Another utility procedure */
+CREATE OR REPLACE FUNCTION delete_entity (entityIDP uuid) RETURNS void
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  numKids INT := (SELECT COUNT(EntityID) FROM filesystem WHERE Parent = entityIDP);
+  isRoot  BOOLEAN := ((SELECT Parent FROM filesystem WHERE EntityID = entityIDP) IS NULL);
+BEGIN
+  /* If this is a directory and has kids raise an error */
+  IF numKids > 0
+  THEN
+    /* entity has children (please dont orphan them O_O ) */
+    RAISE EXCEPTION SQLSTATE '90001' USING MESSAGE = 'entity has children (please dont orphan them O_O )';
+  END IF;
+
+  IF isRoot THEN
+    /* stop trying to delete root >:( */
+    RAISE EXCEPTION SQLSTATE '90001' USING MESSAGE = 'stop trying to delete root >:(';
+  END IF;
+
+  DELETE FROM filesystem WHERE EntityID = entityIDP;
+END $$;
+
+
+
+
+COMMIT;

sqitch/deploy/06-create_dummy_data.sql

@@ -0,0 +1,68 @@
+-- Deploy website:06-create_dummy_data to pg
+-- requires: 05-create_filesystem_table
+
+BEGIN;
+
+-- XXX Add DDLs here.
+SET timezone = 'Australia/Sydney';
+
+/* Create default groups */
+INSERT INTO groups (Name, Permission) VALUES ('admin', 'delete');
+INSERT INTO groups (name, Permission) VALUES ('user', 'write');
+
+/* Setup FS table and modify constraints */
+/* Insert root directory and then add our constraints */
+DO $$
+DECLARE 
+  randomGroup groups.UID%type;
+  rootID      filesystem.EntityID%type;
+BEGIN
+  SELECT groups.UID INTO randomGroup FROM groups WHERE Name = 'admin'::VARCHAR;
+  /* Insert the root directory */
+  INSERT INTO filesystem (EntityID, LogicalName, OwnedBy)
+    VALUES (uuid_nil(), 'root', randomGroup);
+  SELECT filesystem.EntityID INTO rootID FROM filesystem WHERE LogicalName = 'root'::VARCHAR;
+  /* Set parent to uuid_nil() because postgres driver has issue supporting NULL values */
+  UPDATE filesystem SET Parent = uuid_nil() WHERE EntityID = rootID;
+
+  /* insert "has parent" constraint*/
+  EXECUTE 'ALTER TABLE filesystem 
+    ADD CONSTRAINT has_parent CHECK (Parent != NULL)';
+END $$;
+
+
+
+/* create a dummy frontend */
+INSERT INTO frontend (FrontendURL) VALUES ('http://localhost:8080'::VARCHAR);
+
+/* Insert dummy data */
+DO $$
+DECLARE
+  rootID        filesystem.EntityID%type;
+  newEntity     filesystem.EntityID%type;
+  wasPopping    filesystem.EntityID%type;
+  oldEntity     filesystem.EntityID%type;
+BEGIN
+  SELECT filesystem.EntityID INTO rootID FROM filesystem WHERE EntityID = uuid_nil();
+
+  newEntity := (SELECT new_entity(rootID, 'downloads'::VARCHAR, 1, false));
+  oldEntity := (SELECT new_entity(rootID, 'documents'::VARCHAR, 1, false));
+
+  wasPopping := (SELECT new_entity(oldEntity, 'cool_document'::VARCHAR, 1, true));
+  wasPopping := (SELECT new_entity(oldEntity, 'cool_document_round_2'::VARCHAR, 1, true));
+  PERFORM delete_entity(wasPopping);
+  wasPopping := (SELECT new_entity(oldEntity, 'cool_document_round_2'::VARCHAR, 1, true));
+END $$;
+
+
+/* inserting two accounts into db */
+DO LANGUAGE plpgsql $$
+BEGIN
+  EXECUTE create_normal_user('[email protected]', 'adam', 'password', 1);
+  EXECUTE create_normal_user('[email protected]', 'john', 'password', 1);
+  EXECUTE create_normal_user('[email protected]', 'jane', 'password', 1);
+END $$;
+
+
+
+COMMIT;

sqitch/revert/01-create_migration_table.sql

@@ -0,0 +1,8 @@
+-- Revert website:01-create_migration_table from pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+DROP TABLE IF EXISTS migrations CASCADE;
+
+COMMIT;

sqitch/revert/02-create_frontend_table.sql

@@ -0,0 +1,8 @@
+-- Revert website:02-create_frontend_table from pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+DROP TABLE IF EXISTS frontend;
+
+COMMIT;

sqitch/revert/03-create_groups_table.sql

@@ -0,0 +1,9 @@
+-- Revert website:03-create_groups_table from pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+DROP TYPE IF EXISTS permissions_enum CASCADE;
+
+DROP TABLE IF EXISTS groups CASCADE;
+COMMIT;