HAWQ-1629. Add ORC format using pluggable storage framework.

configure

@@ -731,6 +731,7 @@ krb_srvtab
 with_krb5
 with_gssapi
 with_pgcrypto
+with_orc
 with_r
 with_java
 with_python
@@ -860,6 +861,7 @@ with_python
 with_java
 with_r
 with_pgcrypto
+with_orc
 with_gssapi
 with_krb5
 with_krb_srvnam
@@ -5834,6 +5836,40 @@ fi
 $as_echo "$with_pgcrypto" >&6; }
 
 
+#
+# Optionally build with orc in pluggable storage framework.
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build orc modules" >&5
+$as_echo_n "checking whether to build orc modules... " >&6; }
+
+pgac_args="$pgac_args with_orc"
+
+
+# Check whether --with-orc was given.
+if test "${with_orc+set}" = set; then :
+  withval=$with_orc;
+  case $withval in
+    yes)
+      :
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-orc option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_orc=yes
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_orc" >&5
+$as_echo "$with_orc" >&6; }
+
+
 #
 # GSSAPI
 #

configure.in

@@ -722,6 +722,15 @@ PGAC_ARG_BOOL(with, pgcrypto, no, [  --with-pgcrypto         build with pgcrypto
 AC_MSG_RESULT([$with_pgcrypto])
 AC_SUBST(with_pgcrypto)
 
+#
+# Optionally build with orc format in pluggable storage framework.
+#
+AC_MSG_CHECKING([whether to build orc modules])
+PGAC_ARG_BOOL(with, orc, yes, [  --with-orc         build with orc format])
+AC_MSG_RESULT([$with_orc])
+AC_SUBST(with_orc)
+#
+
 #
 # GSSAPI
 #

contrib/Makefile

@@ -10,12 +10,16 @@ WANTED_DIRS = \
 		gp_cancel_query \
 		formatter_fixedwidth \
 		exthdfs\
-		hawq-hadoop
+		hawq-hadoop\
 
 ifeq ($(with_pgcrypto), yes)
 WANTED_DIRS += pgcrypto
 endif
 
+ifeq ($(with_orc), yes)
+WANTED_DIRS += orc
+endif
+
 # Missing:
 #		start-scripts	\ (does not have a makefile)
 #		xml2		\ (requires libxml installed)

contrib/exthdfs/exthdfs.c

@@ -251,13 +251,13 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS)
 			}
 		}
 	}
-	/*if(1)
+	if(1)
 	{
 		ereport(ERROR,
 						(errcode(ERRCODE_SYNTAX_ERROR),
 						 errmsg("hdfsprotocol_validate : "
 								"no formatter is supported for external hdfs")));
-	}*/
+	}
 	if (!isCsv && !isText && !isOrc)
 	{
 		ereport(ERROR,

contrib/orc/Makefile

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+MODULE_big = orc
+OBJS       = orc.o
+
+PG_CPPFLAGS = -I$(libpq_srcdir)
+PG_LIBS = $(libpq_pgport)
+
+
+ifdef USE_PGXS
+PGXS := $(shell pg_config --pgxs)
+include $(PGXS)
+else
+subdir = contrib/orc
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

contrib/orc/README

@@ -0,0 +1,5 @@
+1. Compile ORC format in pluggable storage framework
+   $ ./configure --with-orc; make -j8; make install
+
+2. Configure and initialize cluster
+   $ hawq init cluster -a

contrib/orc/orc.c

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+#include <json-c/json.h>
+
+#include "c.h"
+#include "port.h"
+#include "postgres.h"
+#include "fmgr.h"
+#include "funcapi.h"
+#include "nodes/pg_list.h"
+#include "utils/memutils.h"
+#include "utils/relcache.h"
+#include "utils/uri.h"
+#include "utils/formatting.h"
+#include "utils/lsyscache.h"
+#include "utils/datetime.h"
+#include "mb/pg_wchar.h"
+#include "commands/defrem.h"
+#include "commands/copy.h"
+#include "access/tupdesc.h"
+#include "access/filesplit.h"
+#include "access/plugstorage.h"
+#include "cdb/cdbvars.h"
+#include "catalog/pg_exttable.h"
+#include "catalog/namespace.h"
+#include "postmaster/identity.h"
+#include "nodes/makefuncs.h"
+#include "nodes/plannodes.h"
+#include "utils/uri.h"
+
+
+#define ORC_TIMESTAMP_EPOCH_JDATE	2457024 /* == date2j(2015, 1, 1) */
+#define MAX_ORC_ARRAY_DIMS        10000
+
+/* Do the module magic dance */
+PG_MODULE_MAGIC;
+
+/* Validators for pluggable storage format ORC */
+PG_FUNCTION_INFO_V1(orc_validate_interfaces);
+PG_FUNCTION_INFO_V1(orc_validate_options);
+PG_FUNCTION_INFO_V1(orc_validate_encodings);
+PG_FUNCTION_INFO_V1(orc_validate_datatypes);
+
+/* Accessors for pluggable storage format ORC */
+PG_FUNCTION_INFO_V1(orc_beginscan);
+PG_FUNCTION_INFO_V1(orc_getnext_init);
+PG_FUNCTION_INFO_V1(orc_getnext);
+PG_FUNCTION_INFO_V1(orc_rescan);
+PG_FUNCTION_INFO_V1(orc_endscan);
+PG_FUNCTION_INFO_V1(orc_stopscan);
+PG_FUNCTION_INFO_V1(orc_insert_init);
+PG_FUNCTION_INFO_V1(orc_insert);
+PG_FUNCTION_INFO_V1(orc_insert_finish);
+
+/* Definitions of validators for pluggable storage format ORC */
+Datum orc_validate_interfaces(PG_FUNCTION_ARGS);
+Datum orc_validate_options(PG_FUNCTION_ARGS);
+Datum orc_validate_encodings(PG_FUNCTION_ARGS);
+Datum orc_validate_datatypes(PG_FUNCTION_ARGS);
+
+/* Definitions of accessors for pluggable storage format ORC */
+Datum orc_beginscan(PG_FUNCTION_ARGS);
+Datum orc_getnext_init(PG_FUNCTION_ARGS);
+Datum orc_getnext(PG_FUNCTION_ARGS);
+Datum orc_rescan(PG_FUNCTION_ARGS);
+Datum orc_endscan(PG_FUNCTION_ARGS);
+Datum orc_stopscan(PG_FUNCTION_ARGS);
+Datum orc_insert_init(PG_FUNCTION_ARGS);
+Datum orc_insert(PG_FUNCTION_ARGS);
+Datum orc_insert_finish(PG_FUNCTION_ARGS);
+
+
+Datum orc_validate_interfaces(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_validate_interfaces has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_options(List *formatOptions,
+ *                      char *formatStr,
+ *                      bool isWritable)
+ */
+Datum orc_validate_options(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_validate_options has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_encodings(char *encodingName)
+ */
+Datum orc_validate_encodings(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_validate_encodings has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_datatypes(TupleDesc tupDesc)
+ */
+Datum orc_validate_datatypes(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_validate_datatypes has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * FileScanDesc
+ * orc_beginscan(ExternalScan *extScan,
+ *               ScanState *scanState,
+ *               Relation relation,
+ *               int formatterType,
+ *               char *formatterName)
+ */
+Datum orc_beginscan(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_beginscan has not be completed, please fill it");
+	PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * ExternalSelectDesc
+ * orc_getnext_init(PlanState *planState,
+ *                  ExternalScanState *extScanState)
+ */
+Datum orc_getnext_init(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_getnext_init has not be completed, please fill it");
+	PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * bool
+ * orc_getnext(FileScanDesc fileScanDesc,
+ *             ScanDirection direction,
+ *             ExternalSelectDesc extSelectDesc,
+ *             ScanState *scanState,
+ *             TupleTableSlot *tupTableSlot)
+ */
+Datum orc_getnext(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_getnext has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_rescan(FileScanDesc scan)
+ */
+Datum orc_rescan(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_rescan has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_endscan(FileScanDesc scan)
+ */
+Datum orc_endscan(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_endscan has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_stopscan(FileScanDesc scan)
+ */
+Datum orc_stopscan(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_stopscan has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+
+/*
+ * ExternalInsertDesc
+ * orc_insert_init(Relation relation,
+ *                 int formatterType,
+ *                 char *formatterName)
+ */
+Datum orc_insert_init(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_insert_init has not be completed, please fill it");
+	PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * Oid
+ * orc_insert(ExternalInsertDesc extInsertDesc,
+ *            TupleTableSlot *tupTableSlot)
+ */
+Datum orc_insert(PG_FUNCTION_ARGS)
+{
+
+	elog(ERROR, "Funtion orc_insert has not be completed, please fill it");
+	PG_RETURN_OID(InvalidOid);
+}
+
+/*
+ * void
+ * orc_insert_finish(ExternalInsertDesc extInsertDesc)
+ */
+Datum orc_insert_finish(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "Funtion orc_insert_finish has not be completed, please fill it");
+	PG_RETURN_VOID();
+}
+