diff --git a/README.md b/README.md index f8449e9..787aae6 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,39 @@ hive.allow-rename-table=true - Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt-spark/issues/new). +### Running tests + +Run a Presto server locally: + +``` +cd docker/ +./init.bash +``` + +If you see errors while about "inconsistent state" while bringing up presto, +you may need to drop and re-create the `public` schema in the hive metastore: +``` +# Example error + +Initialization script hive-schema-2.3.0.postgres.sql +Error: ERROR: relation "BUCKETING_COLS" already exists (state=42P07,code=0) +org.apache.hadoop.hive.metastore.HiveMetaException: Schema initialization FAILED! Metastore state would be inconsistent !! +Underlying cause: java.io.IOException : Schema script failed, errorcode 2 +Use --verbose for detailed stacktrace. +*** schemaTool failed *** +``` + +**Solution:** Drop (or rename) the public schema to allow the init script to recreate the metastore from scratch. **Only run this against a test Presto deployment. Do not run this in production!** +```sql +-- run this against the hive metastore (port forwarded to 10005 by default) +-- DO NOT RUN THIS IN PRODUCTION! + +drop schema public cascade; +create schema public; +``` + +You probably should be slightly less reckless than this. + ## Code of Conduct Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/). diff --git a/dbt/adapters/presto/connections.py b/dbt/adapters/presto/connections.py index ffec920..61760b9 100644 --- a/dbt/adapters/presto/connections.py +++ b/dbt/adapters/presto/connections.py @@ -145,7 +145,7 @@ class PrestoConnectionManager(SQLConnectionManager): TYPE = 'presto' @contextmanager - def exception_handler(self, sql, connection_name='master'): + def exception_handler(self, sql): try: yield # TODO: introspect into `DatabaseError`s and expose `errorName`, @@ -155,14 +155,14 @@ def exception_handler(self, sql, connection_name='master'): logger.debug(exc) raise RuntimeException(to_string(exc)) - def add_begin_query(self, name): - connection = self.get(name) - with self.exception_handler('handle.start_transaction()', name): + def add_begin_query(self): + connection = self.get_thread_connection() + with self.exception_handler('handle.start_transaction()'): connection.handle.start_transaction() - def add_commit_query(self, name): - connection = self.get(name) - with self.exception_handler('handle.commit()', name): + def add_commit_query(self): + connection = self.get_thread_connection() + with self.exception_handler('handle.commit()'): connection.handle.commit() @classmethod @@ -200,7 +200,7 @@ def get_status(cls, cursor): def cancel(self, connection): connection.handle.cancel() - def add_query(self, sql, model_name=None, auto_begin=True, + def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False): connection = None @@ -224,7 +224,7 @@ def add_query(self, sql, model_name=None, auto_begin=True, parent = super(PrestoConnectionManager, self) connection, cursor = parent.add_query( - individual_query, model_name, auto_begin, bindings, + individual_query, auto_begin, bindings, abridge_sql_log ) @@ -233,13 +233,12 @@ def add_query(self, sql, model_name=None, auto_begin=True, "Tried to run an empty query on model '{}'. If you are " "conditionally running\nsql, eg. in a model hook, make " "sure your `else` clause contains valid sql!\n\n" - "Provided SQL:\n{}".format(model_name, sql)) + "Provided SQL:\n{}".format(connection.name, sql)) return connection, cursor - def execute(self, sql, name=None, auto_begin=False, fetch=False): - self.get(name) - _, cursor = self.add_query(sql, name, auto_begin) + def execute(self, sql, auto_begin=False, fetch=False): + _, cursor = self.add_query(sql, auto_begin) status = self.get_status(cursor) table = self.get_result_from_cursor(cursor) return status, table diff --git a/dbt/include/presto/macros/adapters.sql b/dbt/include/presto/macros/adapters.sql index 184fcd9..a78d7a8 100644 --- a/dbt/include/presto/macros/adapters.sql +++ b/dbt/include/presto/macros/adapters.sql @@ -94,3 +94,23 @@ {% macro presto__load_csv_rows(model) %} {{ return(basic_load_csv_rows(model, 1000)) }} {% endmacro %} + + +{% macro presto__list_schemas(database) -%} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select distinct schema_name + from {{ information_schema_name(database) }}.schemata + {% endcall %} + {{ return(load_result('list_schemas').table) }} +{% endmacro %} + + +{% macro presto__check_schema_exists(information_schema, schema) -%} + {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%} + select count(*) + from {{ information_schema }}.schemata + where {{ presto_ilike('catalog_name', information_schema.database) }} + and {{ presto_ilike('schema_name', schema) }} + {%- endcall %} + {{ return(load_result('check_schema_exists').table) }} +{% endmacro %} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 549f33b..5082541 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -34,6 +34,8 @@ services: hive-metastore-db: image: postgres + ports: + - "10005:5432" environment: POSTGRES_USER: "root" POSTGRES_PASSWORD: "password" diff --git a/setup.py b/setup.py index ea3ce8c..1f6030e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup package_name = "dbt-presto" -package_version = "0.13.0rc1" +package_version = "0.14.0" description = """The presto adpter plugin for dbt (data build tool)""" setup(