Building Babelfish distribution

Signed-off-by: 3manuek <[email protected]>

.github/workflows/jdbc_unit_tests.yml

@@ -0,0 +1,155 @@
+name: JDBC Unit Tests
+on:
+  push:
+    branches:
+      - BABEL_1_X_DEV__PG_13_4
+  pull_request:
+    branches:
+      - BABEL_1_X_DEV__PG_13_4
+
+env:
+  ANTLR4_VERSION: 4.9.3
+
+jobs:
+  extension-tests:
+    name: Build and test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Requirements
+        run: |
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - &&  \
+          curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
+          sudo apt-get update && sudo apt install -y --no-install-recommends \
+          build-essential flex libxml2-dev libxml2-utils \
+          libxslt-dev libssl-dev \
+          libreadline-dev zlib1g-dev libldap2-dev libpam0g-dev gettext  \
+          uuid uuid-dev cmake lld apt-utils \
+          libossp-uuid-dev gnulib bison \
+          xsltproc icu-devtools libicu66 libicu-dev gawk curl \
+          openjdk-8-jre openssl g++ \
+          libssl-dev python-dev libpq-dev \
+          pkg-config unzip libutfcpp-dev gnupg mssql-tools unixodbc-dev
+          export PATH=/opt/mssql-tools/bin:$PATH
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '8'
+          check-latest: true
+      - name: Copy ANTLR jar file
+        run: |
+          cd contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
+          sudo cp antlr-${ANTLR4_VERSION}-complete.jar /usr/local/lib
+      - name: Compile ANTLR
+        run: |
+          cd ..
+          wget http://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip
+          unzip -d antlr4 antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip 
+          cd antlr4
+          mkdir build && cd build 
+          cmake .. -D ANTLR_JAR_LOCATION=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True
+          make -j 4
+          sudo make install
+          # cp /usr/local/lib/libantlr4-runtime.so.${ANTLR4_VERSION} ~/postgres/lib/
+      - name: Build, and binary installation
+        run: |
+          # CFLAGS="${CFLAGS:--Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic}"
+          ./configure CFLAGS="-ggdb" \
+            --prefix=$HOME/postgres/ \
+            --enable-debug \
+            --with-ldap \
+            --with-libxml \
+            --with-pam \
+            --with-uuid=ossp \
+            --enable-nls \
+            --with-libxslt \
+            --with-icu \
+            --with-extra-version=" Babelfish for PostgreSQL"
+          # ./configure --prefix=$HOME/postgres/ --with-python PYTHON=/usr/bin/python2.7 --enable-debug CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu
+          make clean && make DESTDIR=~/postgres/ -j 4 2>error.txt
+          # make check
+          sudo make install
+      - name: Build antlr
+        run: |
+          export ANTLR4_JAVA_BIN=/usr/bin/java
+          export ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime
+          export ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar
+          export ANTLR_RUNTIME=../antlr4
+          #PG_SRC=~/work/postgresql_modified_for_babelfish
+          export PG_SRC=/home/runner/work/postgresql_modified_for_babelfish/postgresql_modified_for_babelfish/
+          export PG_CONFIG=~/postgres/bin/pg_config
+          cmake=$(which cmake)
+
+          # Copy runtime in Postgres lib
+          sudo cp /usr/local/lib/libantlr4-runtime.so.${ANTLR4_VERSION} ~/postgres/lib
+           
+          cd ${PG_SRC}/contrib/babelfishpg_tsql/antlr 
+          cmake -Wno-dev .
+      - name: Compile and Install Extensions
+        run: |
+          export ANTLR4_JAVA_BIN=/usr/bin/java
+          export ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime
+          export ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar
+          export ANTLR_RUNTIME=../antlr4
+          export PG_SRC=/home/runner/work/postgresql_modified_for_babelfish/postgresql_modified_for_babelfish/
+          export PG_CONFIG=~/postgres/bin/pg_config
+          cmake=$(which cmake)
+
+          cd $PG_SRC/contrib/ && make && sudo make install 
+      - name: Install extensions
+        run: |
+          cd ~
+          sudo ~/postgres/bin/initdb -D ~/postgres/data/ -E "UTF8"
+          chown -R runner: ~/postgres/data/
+          ~/postgres/bin/pg_ctl -D ~/postgres/data/ -l logfile start
+          cd postgres/data
+          sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
+          sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds'/g" postgresql.conf
+          ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}')
+          sudo echo "host    all             all             $ipaddress/32            trust" >> pg_hba.conf
+          ~/postgres/bin/pg_ctl -D ~/postgres/data/ -l logfile restart
+          sudo ~/postgres/bin/psql -d postgres -U runner -c "CREATE USER jdbc_user WITH SUPERUSER CREATEDB CREATEROLE PASSWORD '12345678' INHERIT;"
+          sudo ~/postgres/bin/psql -d postgres -U runner -c "DROP DATABASE IF EXISTS jdbc_testdb;"
+          sudo ~/postgres/bin/psql -d postgres -U runner -c "CREATE DATABASE jdbc_testdb OWNER jdbc_user;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "set allow_system_table_mods = on;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "CREATE EXTENSION IF NOT EXISTS "babelfishpg_tds" CASCADE;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "GRANT ALL ON SCHEMA sys to jdbc_user;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "ALTER USER jdbc_user CREATEDB;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "ALTER SYSTEM SET babelfishpg_tsql.database_name = 'jdbc_testdb';"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "ALTER SYSTEM SET babelfishpg_tds.set_db_session_property = true;"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "SELECT pg_reload_conf();"
+          sudo ~/postgres/bin/psql -d jdbc_testdb -U runner -c "CALL sys.initialize_babelfish('jdbc_user');"
+          sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
+      - name: Run JDBC test framework
+        timeout-minutes: 15
+        run: |
+          cd contrib/test/JDBC/
+          mvn test
+      - name: Upload log
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: postgres-log
+          path: ~/postgres/data/logfile
+      # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
+      - name: Rename test summary files
+        if: ${{ failure() }}
+        run: |
+          cd contrib/test/JDBC/Info
+          timestamp=`ls -Art | tail -n 1`
+          cd $timestamp
+          mv $timestamp.diff ../output-diff.diff
+          mv "$timestamp"_runSummary.log ../run-summary.log
+      - name: Upload run summary 
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: run-summary.log
+          path: contrib/test/JDBC/Info/run-summary.log
+      - name: Upload output diff
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: output-diff.diff
+          path: contrib/test/JDBC/Info/output-diff.diff

INSTALLING

@@ -0,0 +1,112 @@
+# Babelfish distribution compilation and installation
+
+### NOTE: these directions assume you are running from the directory that you
+### uncompressed the Babelfish source code into.
+### By default that would be babel_1.0.0__pg_13.4
+
+export MAX_JOBS=2 # Or any other value suitable for your CPU capacity
+
+export PREFIX=/opt/target/babelfishpg
+export ANTLR4_VERSION=4.9.3
+
+export PG_CONFIG=${PREFIX}/bin/pg_config
+export PG_SRC=$(pwd)
+
+export ANTLR4_RUNTIME_INCLUDE_DIR=/usr/local/include/antlr4-runtime/
+
+export ANTLR4_JAVA_BIN=/usr/bin/java
+
+
+## Prerequisites
+
+apt update
+
+apt install -y --no-install-recommends \
+    build-essential flex libxml2-dev libxml2-utils\
+    libxslt-dev libssl-dev \
+    libreadline-dev zlib1g-dev libldap2-dev libpam0g-dev gettext  \
+    uuid uuid-dev cmake lld apt-utils pkg-config libossp-uuid-dev gnulib bison git
+
+apt install -y --no-install-recommends \
+    xsltproc icu-devtools libicu66 libicu-dev gawk curl
+
+apt install -y openjdk-8-jre openssl \
+    libssl-dev python-dev libpq-dev \
+    pkgconf unzip libutfcpp-dev gnupg
+
+## Antlr compilation
+
+Compiling and installing antlr runtime
+
+ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar
+
+curl https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \
+        --output ${ANTLR_EXECUTABLE} && chmod +x ${ANTLR_EXECUTABLE}
+
+mkdir -p antlr_runtime
+
+curl https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \
+        --output /opt/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip && \
+        unzip -d antlr_runtime /opt/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip
+
+cd antlr_runtime
+mkdir build && cd build && \
+    cmake .. -D ANTLR_JAR_LOCATION=${ANTLR_EXECUTABLE} \
+        -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True && \
+    make && make install
+
+## Compiling Babelfish
+
+
+./configure CFLAGS="${CFLAGS:--Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic}" \
+  --prefix=${PREFIX} \
+  --enable-debug \
+  --with-ldap \
+  --with-libxml \
+  --with-pam \
+  --with-uuid=ossp \
+  --enable-nls \
+  --with-libxslt \
+  --with-icu \
+  --with-extra-version=" Babelfish for PostgreSQL"
+
+# Compilation
+
+make clean && make DESTDIR=${PREFIX} -j ${MAX_JOBS} 
+
+#world-bin
+
+# Install core
+
+make install
+
+# Compile ANTLR shared object
+
+cp /usr/local/lib/libantlr4-runtime.so.${ANTLR4_VERSION} ${PREFIX}/lib
+
+cmake=$(which cmake) ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime 
+cd contrib/babelfishpg_tsql/antlr
+cmake .
+
+# Build all the extensions
+cd ../.. && make && make install && cd ..
+
+
+
+## Starting Babelfish
+
+### For starting and initiating Babelfish, follow up the instructions at
+### https://babelfishpg.org/docs/installation/compiling-babelfish-from-source/#additional-installation-steps).
+
+
+## Clients
+
+### SQL Server Tooling dependencies
+# Reference: https://docs.microsoft.com/en-us/sql/linux/sql-server-linux-setup-tools?view=sql-server-ver15#ubuntu
+# mssql-cli on arm currently isn't supported https://github.com/dbcli/mssql-cli/issues/152#issuecomment-446311124
+# For arm platform, you may find useful: pip install --upgrade mssql-cli
+curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
+curl -L https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list && \
+apt update && ACCEPT_EULA=Y apt install -y mssql-tools unixodbc-dev
+
+PATH="${PREFIX}/bin:/opt/mssql-tools/bin/:${PATH}"

contrib/Makefile

@@ -9,6 +9,10 @@ SUBDIRS = \
 		amcheck		\
 		auth_delay	\
 		auto_explain	\
+		babelfishpg_common \
+		babelfishpg_money \
+		babelfishpg_tds \
+		babelfishpg_tsql \
 		bloom		\
 		btree_gin	\
 		btree_gist	\
@@ -50,6 +54,8 @@ SUBDIRS = \
 		unaccent	\
 		vacuumlo
 
+SUBDIRS += babelfishpg_tsql 
+
 ifeq ($(with_openssl),yes)
 SUBDIRS += sslinfo
 else
@@ -89,6 +95,14 @@ endif
 # Missing:
 #		start-scripts	\ (does not have a makefile)
 
-
 $(recurse)
 $(recurse_always)
+
+
+# all: $(SUBDIRS)
+# $(SUBDIRS):
+# 	$(MAKE) -C $@
+
+# .PHONY: all $(SUBDIRS)
+
+

contrib/babelfishpg_common/Makefile

@@ -0,0 +1,72 @@
+include Version.config
+
+EXTENSION = babelfishpg_common
+EXTVERSION = $(BBFPGCMN_MAJOR_VERSION).$(BBFPGCMN_MINOR_VERSION).$(BBFPGCMN_MICRO_VERSION)
+
+MODULEPATH = $$libdir/$(EXTENSION)-$(BBFPGCMN_MAJOR_VERSION)
+MODULE_big = $(EXTENSION)
+
+PG_CFLAGS += -g
+
+ifdef PREV_EXTVERSION
+DATA = sql/$(EXTENSION)--$(PREV_EXTVERSION).sql
+endif
+
+DATA_built = \
+    $(EXTENSION).control \
+    sql/$(EXTENSION)--$(EXTVERSION).sql
+
+OBJS  = src/babelfishpg_common.o
+OBJS += src/varchar.o
+OBJS += src/bit.o
+OBJS += src/instr.o
+OBJS += src/typecode.o
+OBJS += src/numeric.o
+OBJS += src/varbinary.o
+OBJS += src/uniqueidentifier.o
+OBJS += src/datetime.o
+OBJS += src/datetime2.o
+OBJS += src/smalldatetime.o
+OBJS += src/datetimeoffset.o
+OBJS += src/sqlvariant.o
+OBJS += src/coerce.o
+
+
+ifdef USE_PGXS
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/babelfishpg_common
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+
+
+MODULEPATH = $$libdir/$(EXTENSION)-$(BBFPGCMN_MAJOR_VERSION)
+
+ifdef PREV_EXTVERSION
+DATA = sql/$(EXTENSION)--$(PREV_EXTVERSION).sql
+endif
+
+DATA_built = \
+    $(EXTENSION).control \
+    sql/$(EXTENSION)--$(EXTVERSION).sql $(UPGRADES)
+
+#include ../Makefile.common
+
+$(EXTENSION).control: $(EXTENSION).control.in
+	cat $< \
+		| sed -e 's|@EXTVERSION@|$(EXTVERSION)|g' \
+		| sed -e 's|@EXTENSION@|$(EXTENSION)|g' \
+		| sed -e 's|@MODULEPATH@|$(MODULEPATH)|g' \
+		> $@
+
+sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).in
+	cpp $< | sed 's/^# /-- /g' > $@
+
+
+CFLAGS = `$(PG_CONFIG) --includedir-server`
+
+$(recurse)

contrib/babelfishpg_common/Version.config

@@ -0,0 +1,4 @@
+BBFPGCMN_MAJOR_VERSION=1
+BBFPGCMN_MINOR_VERSION=0
+BBFPGCMN_MICRO_VERSION=0
+

contrib/babelfishpg_common/babelfishpg_common.control.in

@@ -0,0 +1,7 @@
+# TSQL Datatype extension
+comment = 'Transact SQL Datatype Support'
+default_version = '1.0.0'
+module_pathname = '$libdir/babelfishpg_common'
+relocatable = true
+superuser = true
+requires = ''