Merge pull request #43 from arenadata/6.6.0-sync

ADBDEV-3637 6.6.0 sync
arenadata · Jun 9, 2023 · bd6a7fc · bd6a7fc
2 parents 0c78f16 + 865f06d
commit bd6a7fc
Show file tree

Hide file tree

Showing 364 changed files with 6,742 additions and 2,550 deletions.
diff --git a/.github/workflows/create-release-on-tag.yml b/.github/workflows/create-release-on-tag.yml
@@ -22,24 +22,18 @@ jobs:
           tag_name: ${{ github.ref }}
           release_name: PXF Version ${{ github.ref }}
           body: |
-            ## 6.5.0 (12/21/2022)
+            ## 6.6.0 (04/06/2023)
 
             ### Enhancements:
 
-            - [#872](https://github.com/greenplum-db/pxf/pull/872) Enable automation tests to run against PXF FDW extension
-            - [#876](https://github.com/greenplum-db/pxf/pull/876) Parquet write array
-            - [#879](https://github.com/greenplum-db/pxf/pull/879) Add new SPLIT_BY_FILE option for json profiles
-            - [#880](https://github.com/greenplum-db/pxf/pull/880) Allow setting JDBC session-level parallel instructions for Oracle
-            - [#885](https://github.com/greenplum-db/pxf/pull/885) Prevent out-of-bounds buffer access
-            - [#886](https://github.com/greenplum-db/pxf/pull/886) Update jackson-databind to 2.13.4.1
-            - [#895](https://github.com/greenplum-db/pxf/pull/895) Parquet read array
-            - [#903](https://github.com/greenplum-db/pxf/pull/903) Bump postgresql from 42.4.1 to 42.4.3
+            - [#949](https://github.com/greenplum-db/pxf/pull/949) Support for fixedwidth formatter with new `*:fixedwidth` PXF profiles
+            - [#954](https://github.com/greenplum-db/pxf/pull/954) Update table options names to not include dash character
+            - [#955](https://github.com/greenplum-db/pxf/pull/955) Bump jackson-databind from 2.13.4.1 to 2.13.4.2 in /automation
 
             ### Bug Fixes:
 
-            - [#875](https://github.com/greenplum-db/pxf/pull/875) Fix projection of boolean qualifiers whose attrs are not in SELECT list
-            - [#879](https://github.com/greenplum-db/pxf/pull/879) Refactor JsonRecordReader and fix incorrect parsing of JSON objects for multi-line JSON
-            - [#897](https://github.com/greenplum-db/pxf/pull/897) Add close connection if statementRead is null
+            - [#940](https://github.com/greenplum-db/pxf/pull/940) Introduced options to handle decimal overflow when writing Parquet files
 
           draft: false
           prerelease: false
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,25 @@
 # Changelog
 
+## 6.6.0 (04/06/2023)
+
+### Enhancements:
+
+- [#949](https://github.com/greenplum-db/pxf/pull/949) Support for fixedwidth formatter with new `*:fixedwidth` PXF profiles
+- [#954](https://github.com/greenplum-db/pxf/pull/954) Update table options names to not include dash character
+- [#955](https://github.com/greenplum-db/pxf/pull/955) Bump jackson-databind from 2.13.4.1 to 2.13.4.2 in /automation
+
+### Bug Fixes:
+
+- [#940](https://github.com/greenplum-db/pxf/pull/940) Introduced options to handle decimal overflow when writing Parquet files
+
+## 6.5.1 (03/19/2023)
+
+### Enhancements:
+
+- [#857](https://github.com/greenplum-db/pxf/pull/857) Changes for supporting PXF for GP7.
+- [#900](https://github.com/greenplum-db/pxf/pull/900) Added error handling for FDW read flow
+- [#943](https://github.com/greenplum-db/pxf/pull/943) Upgrade gp-common-go-libs to 1.0.11
+
 ## 6.5.0 (12/21/2022)
 
 ### Enhancements:

diff --git a/Makefile b/Makefile
@@ -1,175 +1,197 @@
 include common.mk
 
-PG_CONFIG = pg_config
+PXF_MODULES = external-table fdw cli server
+export PXF_MODULES
 
 PXF_VERSION ?= $(shell cat version)
 export PXF_VERSION
 
-FDW_SUPPORT = $(shell $(PG_CONFIG) --version | egrep "PostgreSQL 12")
+PG_CONFIG ?= pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+ifndef PGXS
+	$(error Make sure the Greenplum installation binaries are in your PATH. i.e. export PATH=<path to your Greenplum installation>/bin:$$PATH)
+endif
+include $(PGXS)
 
-SOURCE_EXTENSION_DIR = external-table
-TARGET_EXTENSION_DIR = gpextable
-ifneq ($(FDW_SUPPORT),)
-	SOURCE_EXTENSION_DIR = fdw
-	TARGET_EXTENSION_DIR = fdw
+# variables that control whether the FDW extension will be built and packaged,
+# if left empty there is no skipping, otherwise a value should contain a reason for skipping
+ifeq ($(shell test $(GP_MAJORVERSION) -lt 6; echo $$?),0)
+	SKIP_FDW_BUILD_REASON := "GPDB version $(GP_MAJORVERSION) is less than 6."
+endif
+ifeq ($(shell test $(GP_MAJORVERSION) -lt 7; echo $$?),0)
+	SKIP_FDW_PACKAGE_REASON := "GPDB version $(GP_MAJORVERSION) is less than 7."
 endif
 
+ifeq ($(BLD_ARCH),)
+	GP_BUILD_ARCH := $(PORTNAME)-$(subst _,-,$(host_cpu))
+else
+	GP_BUILD_ARCH := $(subst _,-,$(BLD_ARCH))
+endif
+
+export SKIP_FDW_BUILD_REASON
+export SKIP_FDW_PACKAGE_REASON
+export GP_MAJORVERSION
+export GP_BUILD_ARCH
+
+PXF_PACKAGE_NAME := pxf-gpdb$(GP_MAJORVERSION)-$(PXF_VERSION)-$(GP_BUILD_ARCH)
+export PXF_PACKAGE_NAME
 
 LICENSE ?= ASL 2.0
 VENDOR ?= Open Source
 
 default: all
 
-.PHONY: all external-table fdw cli server install install-server stage tar rpm rpm-tar deb deb-tar clean test it help
+.PHONY: all extensions external-table fdw cli server install install-server stage tar rpm rpm-tar deb deb-tar clean test it help
+
+all: extensions cli server
+	@echo "===> PXF compilation is complete <==="
 
-all: fdw external-table cli server
+extensions: external-table fdw
 
-external-table:
-	make -C external-table
+external-table cli server:
+	@echo "===> Compiling [$@] module <==="
+	make -C $@
 
 fdw:
-ifneq ($(FDW_SUPPORT),)
+ifeq ($(SKIP_FDW_BUILD_REASON),)
+	@echo "===> Compiling [$@] module <==="
 	make -C fdw
 else
-	@echo "The variable FDW_SUPPORT was empty; skipping building the fdw/ directory"
+	@echo "Skipping building FDW extension because $(SKIP_FDW_BUILD_REASON)"
 endif
 
-cli:
-	make -C cli/go/src/pxf-cli
-
-server:
-	make -C server
-
 clean:
 	rm -rf build
-	make -C $(SOURCE_EXTENSION_DIR) clean-all
-	make -C cli/go/src/pxf-cli clean
-	make -C server clean
+	set -e ;\
+	for module in $${PXF_MODULES[@]}; do \
+		echo "===> Cleaning [$${module}] module <===" ;\
+		make -C $${module} clean-all ;\
+	done ;\
+	echo "===> PXF cleaning is complete <==="
 
 test:
-ifneq ($(FDW_SUPPORT),)
-	make -C fdw test
+ifeq ($(SKIP_FDW_BUILD_REASON),)
+	make -C fdw installcheck
 else
-	@echo "The variable FDW_SUPPORT was empty; skipping fdw tests"
+	@echo "Skipping testing FDW extension because $(SKIP_FDW_BUILD_REASON)"
 endif
-	make -C cli/go/src/pxf-cli test
+	make -C cli test
 	make -C server test
 
 it:
 	make -C automation TEST=$(TEST)
 
 install:
-	make -C $(SOURCE_EXTENSION_DIR) install
-	make -C cli/go/src/pxf-cli install
-	make -C server install
+ifneq ($(SKIP_FDW_BUILD_REASON),)
+	@echo "Skipping installing FDW extension because $(SKIP_FDW_BUILD_REASON)"
+	$(eval PXF_MODULES := $(filter-out fdw,$(PXF_MODULES)))
+endif
+	set -e ;\
+	for module in $${PXF_MODULES[@]}; do \
+		echo "===> Installing [$${module}] module <===" ;\
+		make -C $${module} install ;\
+	done ;\
+	echo "===> PXF installation is complete <==="
 
 install-server:
 	make -C server install-server
 
 stage:
 	rm -rf build/stage
-	make -C $(SOURCE_EXTENSION_DIR) stage
-	make -C cli/go/src/pxf-cli stage
-	make -C server stage
+ifneq ($(SKIP_FDW_PACKAGE_REASON),)
+	@echo "Skipping staging FDW extension because $(SKIP_FDW_PACKAGE_REASON)"
+	$(eval PXF_MODULES := $(filter-out fdw,$(PXF_MODULES)))
+endif
 	set -e ;\
-	GP_MAJOR_VERSION=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/gp_major_version) ;\
-	GP_BUILD_ARCH=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/build_arch) ;\
-	PXF_PACKAGE_NAME=pxf-gpdb$${GP_MAJOR_VERSION}-$${PXF_VERSION}-$${GP_BUILD_ARCH} ;\
-	mkdir -p build/stage/$${PXF_PACKAGE_NAME} ;\
-	cp -a $(SOURCE_EXTENSION_DIR)/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
-	cp -a cli/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
-	cp -a server/build/stage/* build/stage/$${PXF_PACKAGE_NAME} ;\
+	mkdir -p build/stage/$${PXF_PACKAGE_NAME}/pxf ;\
+	for module in $${PXF_MODULES[@]}; do \
+		echo "===> Staging [$${module}] module <===" ;\
+		make -C $${module} stage ;\
+		cp -a "$${module}"/build/stage/* "build/stage/$${PXF_PACKAGE_NAME}/pxf" ;\
+	done ;\
 	echo $$(git rev-parse --verify HEAD) > build/stage/$${PXF_PACKAGE_NAME}/pxf/commit.sha ;\
-	cp package/install_binary build/stage/$${PXF_PACKAGE_NAME}/install_component
+	cp package/install_binary build/stage/$${PXF_PACKAGE_NAME}/install_component ;\
+	echo "===> PXF staging is complete <==="
 
 tar: stage
 	rm -rf build/dist
 	mkdir -p build/dist
-	tar -czf build/dist/$(shell ls build/stage).tar.gz -C build/stage $(shell ls build/stage)
+	tar -czf build/dist/$(PXF_PACKAGE_NAME).tar.gz -C build/stage $(PXF_PACKAGE_NAME)
+	echo "===> PXF TAR file with binaries creation is complete <==="
 
-rpm:
-	make -C $(SOURCE_EXTENSION_DIR) stage
-	make -C cli/go/src/pxf-cli stage
-	make -C server stage
+rpm: stage
+	rm -rf build/rpmbuild
 	set -e ;\
-	GP_MAJOR_VERSION=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/gp_major_version) ;\
 	PXF_MAIN_VERSION=$${PXF_VERSION//-SNAPSHOT/} ;\
 	if [[ $${PXF_VERSION} == *"-SNAPSHOT" ]]; then PXF_RELEASE=SNAPSHOT; else PXF_RELEASE=1; fi ;\
-	rm -rf build/rpmbuild ;\
 	mkdir -p build/rpmbuild/{BUILD,RPMS,SOURCES,SPECS} ;\
-	mkdir -p build/rpmbuild/SOURCES/$(TARGET_EXTENSION_DIR) ;\
-	cp -a $(SOURCE_EXTENSION_DIR)/build/stage/* build/rpmbuild/SOURCES/$(TARGET_EXTENSION_DIR) ;\
-	cp -a cli/build/stage/pxf/* build/rpmbuild/SOURCES ;\
-	cp -a server/build/stage/pxf/* build/rpmbuild/SOURCES ;\
-	echo $$(git rev-parse --verify HEAD) > build/rpmbuild/SOURCES/commit.sha ;\
+	cp -a build/stage/$${PXF_PACKAGE_NAME}/pxf/* build/rpmbuild/SOURCES ;\
 	cp package/*.spec build/rpmbuild/SPECS/ ;\
 	rpmbuild \
 	--define "_topdir $${PWD}/build/rpmbuild" \
 	--define "pxf_version $${PXF_MAIN_VERSION}" \
 	--define "pxf_release $${PXF_RELEASE}" \
 	--define "license ${LICENSE}" \
 	--define "vendor ${VENDOR}" \
-	-bb $${PWD}/build/rpmbuild/SPECS/pxf-gp$${GP_MAJOR_VERSION}.spec
+	-bb $${PWD}/build/rpmbuild/SPECS/pxf-gp$${GP_MAJORVERSION}.spec ;\
+	echo "===> PXF RPM package creation is complete <==="
 
 rpm-tar: rpm
 	rm -rf build/{stagerpm,distrpm}
 	mkdir -p build/{stagerpm,distrpm}
 	set -e ;\
-	GP_MAJOR_VERSION=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/gp_major_version) ;\
-	PXF_RPM_FILE=$$(find build/rpmbuild/RPMS -name pxf-gp$${GP_MAJOR_VERSION}-*.rpm) ;\
+	PXF_RPM_FILE=$$(find build/rpmbuild/RPMS -name pxf-gp$${GP_MAJORVERSION}-*.rpm) ;\
 	PXF_RPM_BASE_NAME=$$(basename $${PXF_RPM_FILE%*.rpm}) ;\
 	PXF_PACKAGE_NAME=$${PXF_RPM_BASE_NAME%.*} ;\
 	mkdir -p build/stagerpm/$${PXF_PACKAGE_NAME} ;\
 	cp $${PXF_RPM_FILE} build/stagerpm/$${PXF_PACKAGE_NAME} ;\
 	cp package/install_rpm build/stagerpm/$${PXF_PACKAGE_NAME}/install_component ;\
-	tar -czf build/distrpm/$${PXF_PACKAGE_NAME}.tar.gz -C build/stagerpm $${PXF_PACKAGE_NAME}
+	tar -czf build/distrpm/$${PXF_PACKAGE_NAME}.tar.gz -C build/stagerpm $${PXF_PACKAGE_NAME} ;\
+	echo "===> PXF TAR file with RPM package creation is complete <==="
 
-deb:
-	make -C $(SOURCE_EXTENSION_DIR) stage
-	make -C cli/go/src/pxf-cli stage
-	make -C server stage
+deb: stage
+	rm -rf build/debbuild
 	set -e ;\
-	GP_MAJOR_VERSION=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/gp_major_version) ;\
 	PXF_MAIN_VERSION=$${PXF_VERSION//-SNAPSHOT/} ;\
 	if [[ $${PXF_VERSION} == *"-SNAPSHOT" ]]; then PXF_RELEASE=SNAPSHOT; else PXF_RELEASE=1; fi ;\
-	rm -rf build/debbuild ;\
-	mkdir -p build/debbuild/usr/local/pxf-gp$${GP_MAJOR_VERSION}/$(TARGET_EXTENSION_DIR) ;\
-	cp -a $(SOURCE_EXTENSION_DIR)/build/stage/* build/debbuild/usr/local/pxf-gp$${GP_MAJOR_VERSION}/$(TARGET_EXTENSION_DIR) ;\
-	cp -a cli/build/stage/pxf/* build/debbuild/usr/local/pxf-gp$${GP_MAJOR_VERSION} ;\
-	cp -a server/build/stage/pxf/* build/debbuild/usr/local/pxf-gp$${GP_MAJOR_VERSION} ;\
-	echo $$(git rev-parse --verify HEAD) > build/debbuild/usr/local/pxf-gp$${GP_MAJOR_VERSION}/commit.sha ;\
+	mkdir -p build/debbuild/usr/local/pxf-gp$${GP_MAJORVERSION} ;\
+	cp -a build/stage/$${PXF_PACKAGE_NAME}/pxf/* build/debbuild/usr/local/pxf-gp$${GP_MAJORVERSION} ;\
 	mkdir build/debbuild/DEBIAN ;\
 	cp -a package/DEBIAN/* build/debbuild/DEBIAN/ ;\
 	sed -i -e "s/%VERSION%/$${PXF_MAIN_VERSION}-$${PXF_RELEASE}/" -e "s/%MAINTAINER%/${VENDOR}/" build/debbuild/DEBIAN/control ;\
 	dpkg-deb --build build/debbuild ;\
-	mv build/debbuild.deb build/pxf-gp$${GP_MAJOR_VERSION}-$${PXF_MAIN_VERSION}-$${PXF_RELEASE}-ubuntu18.04-amd64.deb
+	mv build/debbuild.deb build/pxf-gp$${GP_MAJORVERSION}-$${PXF_MAIN_VERSION}-$${PXF_RELEASE}-ubuntu18.04-amd64.deb ;\
+	echo "===> PXF DEB package creation is complete <==="
+
 
 deb-tar: deb
 	rm -rf build/{stagedeb,distdeb}
 	mkdir -p build/{stagedeb,distdeb}
 	set -e ;\
-	GP_MAJOR_VERSION=$$(cat $(SOURCE_EXTENSION_DIR)/build/metadata/gp_major_version) ;\
-	PXF_DEB_FILE=$$(find build/ -name pxf-gp$${GP_MAJOR_VERSION}*.deb) ;\
+	PXF_DEB_FILE=$$(find build/ -name pxf-gp$${GP_MAJORVERSION}*.deb) ;\
 	PXF_PACKAGE_NAME=$$(dpkg-deb --field $${PXF_DEB_FILE} Package)-$$(dpkg-deb --field $${PXF_DEB_FILE} Version)-ubuntu18.04 ;\
 	mkdir -p build/stagedeb/$${PXF_PACKAGE_NAME} ;\
 	cp $${PXF_DEB_FILE} build/stagedeb/$${PXF_PACKAGE_NAME} ;\
 	cp package/install_deb build/stagedeb/$${PXF_PACKAGE_NAME}/install_component ;\
-	tar -czf build/distdeb/$${PXF_PACKAGE_NAME}.tar.gz -C build/stagedeb $${PXF_PACKAGE_NAME}
+	tar -czf build/distdeb/$${PXF_PACKAGE_NAME}.tar.gz -C build/stagedeb $${PXF_PACKAGE_NAME} ;\
+	echo "===> PXF TAR file with DEB package creation is complete <==="
 
 
 help:
 	@echo
 	@echo 'Possible targets'
-	@echo	'  - all (external-table, cli, server)'
+	@echo	'  - all - build extensions, cli, and server modules'
+	@echo	'  - extensions - build Greenplum external table and foreign data wrapper extensions'
 	@echo	'  - external-table - build Greenplum external table extension'
-	@echo	'  - fdw - build PXF Foreign-Data Wrapper Greenplum extension'
+	@echo	'  - fdw - build Greenplum foreign data wrapper extension'
 	@echo	'  - cli - install Go CLI dependencies and build Go CLI'
-	@echo	'  - server - install PXF server dependencies and build PXF server'
+	@echo	'  - server - install server dependencies and build server module'
 	@echo	'  - clean - clean up external-table, fdw, CLI and server binaries'
-	@echo	'  - test - runs tests for PXF Go CLI and server'
-	@echo	'  - install - install PXF external table extension, CLI and server'
-	@echo	'  - install-server - install PXF server without running tests'
-	@echo	'  - tar - bundle PXF external table extension, CLI, server and tomcat into a single tarball'
+	@echo	'  - test - runs tests for Go CLI and server'
+	@echo	'  - install - install external table and foreign data wrapper extensions, CLI and server binaries'
+	@echo	'  - install-server - install server binaries only without running tests'
+	@echo	'  - stage - install external table and foreign data wrapper extensions, CLI, and server binaries into build/stage/pxf directory'
+	@echo	'  - tar - bundle external table and foreign data wrapper extensions, CLI, and server into a single tarball'
 	@echo	'  - rpm - create PXF RPM package'
 	@echo	'  - rpm-tar - bundle PXF RPM package along with helper scripts into a single tarball'
 	@echo	'  - deb - create PXF DEB package'

diff --git a/README.md b/README.md
@@ -92,14 +92,6 @@ To build PXF, you must have:
    export PATH=$PATH:/opt/homebrew/bin/go/bin:$GOPATH/bin
    ```
 
-    Once you have installed Go, you will need the `ginkgo` tool which runs Go tests,
-    respectively. Assuming `go` is on your `PATH`, you can run:
-    ```shell
-    go install github.com/onsi/ginkgo/[email protected]
-    ```
-
-   * NOTE: You are required to list the version. For now, keep it stable at v1.16.5 which is what we use for the concourse images. See `concourse/docker/pxf-dev-base/cloudbuild.yml` for the default values.
-
 5. cURL (7.29 or later):
 
     To install cURL devel package on CentOS 7, `sudo yum install libcurl-devel`.

diff --git a/arenadata/Dockerfile b/arenadata/Dockerfile
@@ -25,6 +25,7 @@ RUN source gpdb_src/concourse/scripts/common.bash && \
     install_gpdb && \
     source '/usr/local/greenplum-db-devel/greenplum_path.sh' && \
     mkdir ${OUTPUT_ARTIFACT_DIR} && \
+    export SKIP_FDW_BUILD_REASON=0 && \
     pxf_src/concourse/scripts/compile_pxf.bash
 
 # create test image which prepares base image and keeps only pxf artifacts from build image

diff --git a/automation/README.md b/automation/README.md
@@ -139,6 +139,14 @@ Note: if you get an error saying that the jar does not exist, ensure that you ha
 a) installed the PXF server, and
 b) only have 1 jar file inside `/usr/local/pxf/application/`
 
+### Running automation tests for GPDB7
+Tinc only runs with Python 2 so it won't work with the Python 3 libraries that come with GP7.
+We use the required Python 2 libraries from GP6 in order to run tinc against GP7.
+
+To support running the tests for GPDB 7, export PXF_AUTOMATION_TINC_DEPS to point to GPDB6 python libs.
+
+`export PXF_AUTOMATION_TINC_DEPS="${GP6HOME}/lib/python"`
+
 ### Project structure
 _**src/main/java**_ - contains related classes and utilities for the test
 

diff --git a/automation/pom.xml b/automation/pom.xml
@@ -209,7 +209,7 @@
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-web</artifactId>
-            <version>5.2.6.RELEASE</version>
+            <version>5.3.18</version>
             <scope>provided</scope>
         </dependency>
 
@@ -247,7 +247,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.4.1</version>
+            <version>2.13.4.2</version>
         </dependency>
 
         <dependency>