Merge pull request #7 from fishtown-analytics/compat/0.14.0

Support for 0.14.0

README.md

@@ -63,6 +63,39 @@ hive.allow-rename-table=true
 
 -   Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt-spark/issues/new).
 
+### Running tests
+
+Run a Presto server locally:
+
+```
+cd docker/
+./init.bash
+```
+
+If you see errors while about "inconsistent state" while bringing up presto,
+you may need to drop and re-create the `public` schema in the hive metastore:
+```
+# Example error
+
+Initialization script hive-schema-2.3.0.postgres.sql
+Error: ERROR: relation "BUCKETING_COLS" already exists (state=42P07,code=0)
+org.apache.hadoop.hive.metastore.HiveMetaException: Schema initialization FAILED! Metastore state would be inconsistent !!
+Underlying cause: java.io.IOException : Schema script failed, errorcode 2
+Use --verbose for detailed stacktrace.
+*** schemaTool failed ***
+```
+
+**Solution:** Drop (or rename) the public schema to allow the init script to recreate the metastore from scratch. **Only run this against a test Presto deployment. Do not run this in production!**
+```sql
+-- run this against the hive metastore (port forwarded to 10005 by default)
+-- DO NOT RUN THIS IN PRODUCTION!
+
+drop schema public cascade;
+create schema public;
+```
+
+You probably should be slightly less reckless than this.
+
 ## Code of Conduct
 
 Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/).

dbt/adapters/presto/connections.py

@@ -145,7 +145,7 @@ class PrestoConnectionManager(SQLConnectionManager):
     TYPE = 'presto'
 
     @contextmanager
-    def exception_handler(self, sql, connection_name='master'):
+    def exception_handler(self, sql):
         try:
             yield
         # TODO: introspect into `DatabaseError`s and expose `errorName`,
@@ -155,14 +155,14 @@ def exception_handler(self, sql, connection_name='master'):
             logger.debug(exc)
             raise RuntimeException(to_string(exc))
 
-    def add_begin_query(self, name):
-        connection = self.get(name)
-        with self.exception_handler('handle.start_transaction()', name):
+    def add_begin_query(self):
+        connection = self.get_thread_connection()
+        with self.exception_handler('handle.start_transaction()'):
             connection.handle.start_transaction()
 
-    def add_commit_query(self, name):
-        connection = self.get(name)
-        with self.exception_handler('handle.commit()', name):
+    def add_commit_query(self):
+        connection = self.get_thread_connection()
+        with self.exception_handler('handle.commit()'):
             connection.handle.commit()
 
     @classmethod
@@ -200,7 +200,7 @@ def get_status(cls, cursor):
     def cancel(self, connection):
         connection.handle.cancel()
 
-    def add_query(self, sql, model_name=None, auto_begin=True,
+    def add_query(self, sql, auto_begin=True,
                   bindings=None, abridge_sql_log=False):
 
         connection = None
@@ -224,7 +224,7 @@ def add_query(self, sql, model_name=None, auto_begin=True,
 
             parent = super(PrestoConnectionManager, self)
             connection, cursor = parent.add_query(
-                individual_query, model_name, auto_begin, bindings,
+                individual_query, auto_begin, bindings,
                 abridge_sql_log
             )
 
@@ -233,13 +233,12 @@ def add_query(self, sql, model_name=None, auto_begin=True,
                     "Tried to run an empty query on model '{}'. If you are "
                     "conditionally running\nsql, eg. in a model hook, make "
                     "sure your `else` clause contains valid sql!\n\n"
-                    "Provided SQL:\n{}".format(model_name, sql))
+                    "Provided SQL:\n{}".format(connection.name, sql))
 
         return connection, cursor
 
-    def execute(self, sql, name=None, auto_begin=False, fetch=False):
-        self.get(name)
-        _, cursor = self.add_query(sql, name, auto_begin)
+    def execute(self, sql, auto_begin=False, fetch=False):
+        _, cursor = self.add_query(sql, auto_begin)
         status = self.get_status(cursor)
         table = self.get_result_from_cursor(cursor)
         return status, table

dbt/include/presto/macros/adapters.sql

@@ -94,3 +94,23 @@
 {% macro presto__load_csv_rows(model) %}
   {{ return(basic_load_csv_rows(model, 1000)) }}
 {% endmacro %}
+
+
+{% macro presto__list_schemas(database) -%}
+  {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
+    select distinct schema_name
+    from {{ information_schema_name(database) }}.schemata
+  {% endcall %}
+  {{ return(load_result('list_schemas').table) }}
+{% endmacro %}
+
+
+{% macro presto__check_schema_exists(information_schema, schema) -%}
+  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%}
+        select count(*)
+        from {{ information_schema }}.schemata
+        where {{ presto_ilike('catalog_name', information_schema.database) }}
+          and {{ presto_ilike('schema_name', schema) }}
+  {%- endcall %}
+  {{ return(load_result('check_schema_exists').table) }}
+{% endmacro %}

docker/docker-compose.yml

@@ -34,6 +34,8 @@ services:
 
   hive-metastore-db:
     image: postgres
+    ports:
+      - "10005:5432"
     environment:
       POSTGRES_USER: "root"
       POSTGRES_PASSWORD: "password"

setup.py

@@ -3,7 +3,7 @@
 from distutils.core import setup
 
 package_name = "dbt-presto"
-package_version = "0.13.0rc1"
+package_version = "0.14.0"
 description = """The presto adpter plugin for dbt (data build tool)"""
 
 setup(