Skip to content

Commit

Permalink
Support ICU and fix Windows builds
Browse files Browse the repository at this point in the history
  • Loading branch information
AjayP13 committed Nov 13, 2018
1 parent b7796a5 commit da43004
Show file tree
Hide file tree
Showing 800 changed files with 461,202 additions and 19 deletions.
65 changes: 47 additions & 18 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,21 @@ def copy_sqlite(src, dest, apsw=False):
outfile.write('#endif\n')


def get_modules(THIRD_PARTY, INTERNAL, PROJ_PATH, SO_SUFFIX):
def get_modules(THIRD_PARTY, INTERNAL, PROJ_PATH,
SO_SUFFIX, source_for_module_with_pyinit):
""" Get all modules this package needs compiled """
PYSQLITE2 = INTERNAL + '/pysqlite2'
APSW = INTERNAL + '/apsw'
PYSQLITE = THIRD_PARTY + '/_pysqlite'
APSW_TP = THIRD_PARTY + '/_apsw'
SQLITE3 = THIRD_PARTY + '/sqlite3'
ICU_UNIX = SQLITE3 + '/icu_unix'
ICU_WIN32 = SQLITE3 + '/icu_win32'
libraries = [os.path.relpath(SQLITE3, PROJ_PATH)]
if sys.platform == 'win32':
libraries.append(ICU_WIN32)
else:
libraries.append(ICU_UNIX)

SQLITE_PRE = os.path.relpath(
os.path.join(SQLITE3, 'sqlite3.c.pre.c'), PROJ_PATH)
Expand Down Expand Up @@ -167,29 +175,32 @@ def get_modules(THIRD_PARTY, INTERNAL, PROJ_PATH, SO_SUFFIX):
with open(SQLITE_PRE, 'r') as infile:
for line in infile:
outfile.write(line)

module = 'sqlite3'
pyinit_source = source_for_module_with_pyinit(module)
sqlite3 = Extension('sqlite3' + SO_SUFFIX,
sources=[SQLITE_POST],
sources=[SQLITE_POST] + [pyinit_source],
include_dirs=[os.path.relpath(SQLITE3, PROJ_PATH)],
library_dirs=[os.path.relpath(SQLITE3, PROJ_PATH)],
library_dirs=libraries,
extra_compile_args=["-O4"],
extra_link_args=["-flto"])

def sqlite_extension(ext, skip=[], name=None):
name = name or ext
def sqlite_extension(ext, skip=[], module=None):
module = module or ext
pyinit_source = source_for_module_with_pyinit(module)
return Extension(
name + SO_SUFFIX,
sources=[
module + SO_SUFFIX,
sources=([
g for g in glob(
os.path.join(
SQLITE_EXT,
ext,
'*.c')) if os.path.basename(g) not in skip],
'*.c')) if os.path.basename(g) not in skip] +
[pyinit_source]),
include_dirs=[
os.path.relpath(
SQLITE3,
PROJ_PATH)],
library_dirs=[os.path.relpath(SQLITE3, PROJ_PATH)],
library_dirs=libraries,
extra_compile_args=["-O4"],
extra_link_args=["-flto"])

Expand All @@ -198,10 +209,13 @@ def sqlite_misc_extensions(skip=[]):
for source in glob(os.path.join(SQLITE_EXT, 'misc', '*.c')):
if os.path.basename(source) in skip:
continue
module = os.path.basename(source)[:-2]
pyinit_source = source_for_module_with_pyinit(module)
miscs.append(
Extension(os.path.basename(source)[:-2] + SO_SUFFIX,
sources=[source],
Extension(module + SO_SUFFIX,
sources=[source] + [pyinit_source],
include_dirs=[os.path.relpath(SQLITE3, PROJ_PATH)],
library_dirs=libraries,
extra_compile_args=["-O4"],
extra_link_args=["-flto"]))
return miscs
Expand All @@ -211,8 +225,8 @@ def sqlite_misc_extensions(skip=[]):
# fts1 = sqlite_extension('fts1') deprecated
# fts2 = sqlite_extension('fts2') deprecated
fts3 = sqlite_extension('fts3', skip=['fts3_test.c'])
fts5 = sqlite_extension('fts5built', name='fts5')
# icu = sqlite_extension('icu') requires a library
fts5 = sqlite_extension('fts5built', module='fts5')
icu = sqlite_extension('icu')
lsm1 = sqlite_extension('lsm1')
rbu = sqlite_extension('rbu')
rtree = sqlite_extension('rtree', skip=['geopoly.c'])
Expand All @@ -221,7 +235,7 @@ def sqlite_misc_extensions(skip=[]):
userauth = sqlite_extension('userauth')

return ([sqlite3, async_m, expert, fts3,
fts5, lsm1, rbu, rtree, session, userauth] +
fts5, icu, lsm1, rbu, rtree, session, userauth] +
sqlite_misc_extensions())


Expand Down Expand Up @@ -386,6 +400,7 @@ def flush(self):
BUILD_THIRD_PARTY = BUILD_PATH + '/lib/' + PACKAGE_NAME + '/third_party'
INTERNAL = THIRD_PARTY + '/internal'
SO_SUFFIX = '00000' + ''.join(format(ord(x), 'b') for x in PACKAGE_NAME)
BINARY_EXTENSIONS = ('.so', '.pyd', '.dll', '.o', '.obj', '.lib')

# Get the package version
__version__ = None
Expand Down Expand Up @@ -431,7 +446,6 @@ def flush(self):
hashlib.md5(PROJ_PATH.encode('utf-8')).hexdigest() +
'.buildext'
)
MODULES = get_modules(THIRD_PARTY, INTERNAL, PROJ_PATH, SO_SUFFIX)


def try_list_dir(d):
Expand Down Expand Up @@ -665,14 +679,26 @@ def get_site_packages():
return []


def source_for_module_with_pyinit(module):
""" Create PyInit symbols for shared objects compiled with Python's
Extension()"""
source_file = os.path.join(tempfile.mkdtemp(), module + '.c')
with open(source_file, 'w+') as outfile:
outfile.write('''
void init''' + (module + SO_SUFFIX) + '''() {} //Python 2.7
void PyInit_''' + (module + SO_SUFFIX) + '''() {} //Python 3.5
''')
return os.path.relpath(source_file, PROJ_PATH)


def delete_shared_objects():
""" Deletes shared object files made with build_ext from site_packages """
site_packages = get_site_packages()
for site_pack in site_packages:
print("Deleting shared objects...", os.path.abspath(site_pack))
for root, dirnames, filenames in list(os.walk(site_pack)):
for filename in filenames:
if filename.lower().endswith(('.so', '.pyd', '.dll', '.o')):
if filename.lower().endswith(BINARY_EXTENSIONS):
so = os.path.join(root, filename)
so_base = os.path.basename(so)
if SO_SUFFIX in so_base:
Expand All @@ -693,7 +719,7 @@ def copy_shared_objects():
print("Copying shared objects...", os.path.abspath(BUILD_PATH))
for root, dirnames, filenames in list(os.walk(BUILD_PATH)):
for filename in filenames:
if filename.lower().endswith(('.so', '.pyd', '.dll', '.o')):
if filename.lower().endswith(BINARY_EXTENSIONS):
so = os.path.join(root, filename)
so_base = os.path.basename(so)
if SO_SUFFIX in so_base:
Expand Down Expand Up @@ -872,6 +898,9 @@ def has_ext_modules(foo):
return True


MODULES = get_modules(THIRD_PARTY, INTERNAL, PROJ_PATH,
SO_SUFFIX, source_for_module_with_pyinit)

if __name__ == '__main__':

# Attempt to install from a remote pre-compiled wheel
Expand Down
235 changes: 235 additions & 0 deletions supersqlite/third_party/sqlite3/icu_unix/Makefile.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
#******************************************************************************
#
# Copyright (C) 1999-2016, International Business Machines
# Corporation and others. All Rights Reserved.
#
#******************************************************************************
## Makefile.in for ICU - icuuc.so
## Stephen F. Booth

## Source directory information
srcdir = @srcdir@
top_srcdir = @top_srcdir@

top_builddir = ..

## All the flags and other definitions are included here.
include $(top_builddir)/icudefs.mk

## Build directory information
subdir = common

# for service hook
LOCALSVC_CPP=localsvc.cpp
SVC_HOOK_INC=$(top_builddir)/common/svchook.mk

## Extra files to remove for 'make clean'
CLEANFILES = *~ $(DEPS) $(IMPORT_LIB) $(MIDDLE_IMPORT_LIB) $(FINAL_IMPORT_LIB) $(SVC_HOOK_INC)

## Target information

TARGET_STUBNAME=$(COMMON_STUBNAME)

ifneq ($(ENABLE_STATIC),)
TARGET = $(LIBDIR)/$(LIBSICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(A)
endif

ifneq ($(ENABLE_SHARED),)
SO_TARGET = $(LIBDIR)/$(LIBICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(SO)
ALL_SO_TARGETS = $(SO_TARGET) $(MIDDLE_SO_TARGET) $(FINAL_SO_TARGET) $(SHARED_OBJECT)

ifeq ($(ENABLE_SO_VERSION_DATA),1)
SO_VERSION_DATA = common.res
endif

ifeq ($(OS390BATCH),1)
BATCH_TARGET = $(BATCH_COMMON_TARGET)
BATCH_LIBS = $(BATCH_LIBICUDT) -lm
endif # OS390BATCH

endif # ENABLE_SHARED

ALL_TARGETS = $(TARGET) $(ALL_SO_TARGETS) $(BATCH_TARGET)

DYNAMICCPPFLAGS = $(SHAREDLIBCPPFLAGS)
DYNAMICCFLAGS = $(SHAREDLIBCFLAGS)
DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS)
CFLAGS += $(LIBCFLAGS)
CXXFLAGS += $(LIBCXXFLAGS)
ifeq ($(OS390BATCH),1)
CFLAGS += -WI
CXXFLAGS += -WI
endif

CPPFLAGS += -I$(srcdir) $(LIBCPPFLAGS) $(CPPFLAGSICUUC)
# we want DEFS here
DEFS += -DU_COMMON_IMPLEMENTATION
LDFLAGS += $(LDFLAGSICUUC)

# for plugin configuration
CPPFLAGS += "-DDEFAULT_ICU_PLUGINS=\"$(libdir)/icu\" "

# for icu data location
ifeq ($(PKGDATA_MODE),common)
CPPFLAGS += "-DU_ICU_DATA_DEFAULT_DIR=\"$(ICUDATA_DIR)\""
endif

# $(LIBICUDT) is either stub data or the real DLL common data.
LIBS = $(LIBICUDT) $(DEFAULT_LIBS)

OBJECTS = errorcode.o putil.o umath.o utypes.o uinvchar.o umutex.o ucln_cmn.o \
uinit.o uobject.o cmemory.o charstr.o cstr.o \
udata.o ucmndata.o udatamem.o umapfile.o udataswp.o utrie_swap.o ucol_swp.o utrace.o \
uhash.o uhash_us.o uenum.o ustrenum.o uvector.o ustack.o uvectr32.o uvectr64.o \
ucnv.o ucnv_bld.o ucnv_cnv.o ucnv_io.o ucnv_cb.o ucnv_err.o ucnvlat1.o \
ucnv_u7.o ucnv_u8.o ucnv_u16.o ucnv_u32.o ucnvscsu.o ucnvbocu.o \
ucnv_ext.o ucnvmbcs.o ucnv2022.o ucnvhz.o ucnv_lmb.o ucnvisci.o ucnvdisp.o ucnv_set.o ucnv_ct.o \
resource.o uresbund.o ures_cnv.o uresdata.o resbund.o resbund_cnv.o \
ucurr.o \
messagepattern.o ucat.o locmap.o uloc.o locid.o locutil.o locavailable.o locdispnames.o locdspnm.o loclikely.o locresdata.o \
bytestream.o stringpiece.o bytesinkutil.o \
stringtriebuilder.o bytestriebuilder.o \
bytestrie.o bytestrieiterator.o \
ucharstrie.o ucharstriebuilder.o ucharstrieiterator.o \
dictionarydata.o \
edits.o \
appendable.o ustr_cnv.o unistr_cnv.o unistr.o unistr_case.o unistr_props.o \
utf_impl.o ustring.o ustrcase.o ucasemap.o ucasemap_titlecase_brkiter.o cstring.o ustrfmt.o ustrtrns.o ustr_wcs.o utext.o \
unistr_case_locale.o ustrcase_locale.o unistr_titlecase_brkiter.o ustr_titlecase_brkiter.o \
normalizer2impl.o normalizer2.o filterednormalizer2.o normlzr.o unorm.o unormcmp.o loadednormalizer2impl.o \
chariter.o schriter.o uchriter.o uiter.o \
patternprops.o uchar.o uprops.o ucase.o propname.o ubidi_props.o characterproperties.o \
ubidi.o ubidiwrt.o ubidiln.o ushape.o \
uscript.o uscript_props.o usc_impl.o unames.o \
utrie.o utrie2.o utrie2_builder.o ucptrie.o umutablecptrie.o \
bmpset.o unisetspan.o uset_props.o uniset_props.o uniset_closure.o uset.o uniset.o usetiter.o ruleiter.o caniter.o unifilt.o unifunct.o \
uarrsort.o brkiter.o ubrk.o brkeng.o dictbe.o filteredbrk.o \
rbbi.o rbbidata.o rbbinode.o rbbirb.o rbbiscan.o rbbisetb.o rbbistbl.o rbbitblb.o rbbi_cache.o \
serv.o servnotf.o servls.o servlk.o servlkf.o servrbf.o servslkf.o \
uidna.o usprep.o uts46.o punycode.o \
util.o util_props.o parsepos.o locbased.o cwchar.o wintz.o dtintrv.o ucnvsel.o propsvec.o \
ulist.o uloc_tag.o icudataver.o icuplug.o \
sharedobject.o simpleformatter.o unifiedcache.o uloc_keytype.o \
ubiditransform.o \
pluralmap.o \
static_unicode_sets.o

## Header files to install
HEADERS = $(srcdir)/unicode/*.h

STATIC_OBJECTS = $(OBJECTS:.o=.$(STATIC_O))

DEPS = $(OBJECTS:.o=.d)

-include Makefile.local

-include $(SVC_HOOK_INC)


## List of phony targets
.PHONY : all all-local install install-local clean clean-local \
distclean distclean-local install-library install-headers dist \
dist-local check check-local check-exhaustive

## Clear suffix list
.SUFFIXES :

## List of standard targets
all: all-local
install: install-local
clean: clean-local
distclean : distclean-local
dist: dist-local
check: all check-local

check-exhaustive: check

all-local: $(ALL_TARGETS)

install-local: install-headers install-library

install-library: all-local
$(MKINSTALLDIRS) $(DESTDIR)$(libdir)
ifneq ($(ENABLE_STATIC),)
$(INSTALL-L) $(TARGET) $(DESTDIR)$(libdir)
endif
ifneq ($(ENABLE_SHARED),)
$(INSTALL-L) $(FINAL_SO_TARGET) $(DESTDIR)$(libdir)
ifneq ($(FINAL_SO_TARGET),$(SO_TARGET))
cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(SO_TARGET))
ifneq ($(FINAL_SO_TARGET),$(MIDDLE_SO_TARGET))
cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(MIDDLE_SO_TARGET))
endif
endif
ifneq ($(IMPORT_LIB_EXT),)
$(INSTALL-L) $(FINAL_IMPORT_LIB) $(DESTDIR)$(libdir)
ifneq ($(IMPORT_LIB),$(FINAL_IMPORT_LIB))
cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(IMPORT_LIB))
endif
ifneq ($(MIDDLE_IMPORT_LIB),$(FINAL_IMPORT_LIB))
cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(MIDDLE_IMPORT_LIB))
endif
endif
endif

$(SVC_HOOK_INC):
@echo generating $@
@-test -f $(top_srcdir)/common/$(LOCALSVC_CPP) && ( echo "have $(LOCALSVC_CPP) - U_LOCAL_SERVICE_HOOK=1" ; \
echo 'CPPFLAGS +=-DU_LOCAL_SERVICE_HOOK=1' > $@ ; \
echo 'OBJECTS += $(LOCALSVC_CPP:%.cpp=%.o)' >> $@ \
) ; true
@echo "# Autogenerated by Makefile" >> $@

install-headers:
$(MKINSTALLDIRS) $(DESTDIR)$(includedir)/unicode
@for file in $(HEADERS); do \
echo "$(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode"; \
$(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode || exit; \
done

dist-local:

clean-local:
test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES)
$(RMV) $(OBJECTS) $(STATIC_OBJECTS) $(ALL_TARGETS) $(SO_VERSION_DATA)

distclean-local: clean-local
$(RMV) Makefile icucfg.h $(SVC_HOOK_INC)

check-local:

Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status $(SVC_HOOK_INC)
cd $(top_builddir) \
&& CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status

ifneq ($(ENABLE_STATIC),)
$(TARGET): $(STATIC_OBJECTS)
$(AR) $(ARFLAGS) $(AR_OUTOPT)$@ $^
$(RANLIB) $@
endif

ifneq ($(ENABLE_SHARED),)
$(SHARED_OBJECT): $(OBJECTS) $(SO_VERSION_DATA)
$(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(LIBS)
ifeq ($(ENABLE_RPATH),YES)
ifneq ($(wildcard $(libdir)/$(MIDDLE_SO_TARGET)),)
$(warning RPATH warning: --enable-rpath means test programs may use existing $(libdir)/$(MIDDLE_SO_TARGET))
endif
endif

ifeq ($(OS390BATCH),1)
$(BATCH_TARGET):$(OBJECTS)
$(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(BATCH_LIBS)
endif # OS390BATCH
endif # ENABLE_SHARED

ifeq (,$(MAKECMDGOALS))
-include $(DEPS)
else
ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),)
-include $(DEPS)
endif
endif

Loading

0 comments on commit da43004

Please sign in to comment.