setup.py

from __future__ import print_function

import hashlib
import os
import re
import requests
import shutil
import subprocess
import sys

from distutils.command.build import build
from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext
from Cython.Build import cythonize


OPENFST_VERSION = "1.7.9"


def copy(src, dst):
    print("copying {} -> {}".format(src, dst))
    shutil.copy(src, dst)


def get_file_sha256(filename):
    hasher = hashlib.md5()
    with open(filename, "rb") as afile:
        buf = afile.read()
        hasher.update(buf)
    return hasher.hexdigest()


def get_filename_with_sha256(filename):
    sha256 = get_file_sha256(filename)
    basename = os.path.basename(filename)
    basename_parts = basename.split(".")
    libname, libext = basename_parts[0], ".".join(basename_parts[1:])
    return "%s-%s.%s" % (libname, sha256[:8], libext)


class OpenFstExtension(Extension):
    def __init__(self):
        Extension.__init__(self, name="openfst_python.pywrapfst", sources=[])


class OpenFstBuild(build):

    user_options = build.user_options + [
        (
            "download-dir=",
            None,
            "directory containing the openfst-%s.tar.gz file" % OPENFST_VERSION,
        )
    ]

    @property
    def openfst_basename(self):
        return "openfst-%s.tar.gz" % OPENFST_VERSION

    def initialize_options(self):
        build.initialize_options(self)
        self.download_dir = None

    def finalize_options(self):
        build.finalize_options(self)
        # self.set_undefined_options("build_ext", ("download_dir", "download_dir"))
        if self.download_dir:
            openfst_tar_gz = os.path.join(self.download_dir, self.openfst_basename)
            assert os.path.isfile(openfst_tar_gz), (
                "File %s does not exist" % openfst_tar_gz
            )
        else:
            self.download_dir = self.build_temp


class OpenFstBuildExt(build_ext):

    user_options = build_ext.user_options + [
        (
            "download-dir=",
            None,
            "directory containing the openfst-%s.tar.gz file" % OPENFST_VERSION,
        )
    ]

    @property
    def openfst_basename(self):
        return "openfst-%s.tar.gz" % OPENFST_VERSION

    @property
    def openfst_dirname(self):
        return "%s/openfst-%s" % (self.build_temp, OPENFST_VERSION)

    @property
    def openfst_filename(self):
        return os.path.join(self.download_dir, self.openfst_basename)

    @property
    def openfst_url(self):
        base_url = "http://www.openfst.org/twiki/pub/FST/FstDownload"
        return "%s/%s" % (base_url, self.openfst_basename)

    @property
    def openfst_main_lib(self):
        return "%s/src/extensions/python/.libs/pywrapfst.so" % self.openfst_dirname

    @property
    def openfst_deps_libs(self):
        return [
            "%s/src/extensions/far/.libs/libfstfar.so" % self.openfst_dirname,
            "%s/src/extensions/far/.libs/libfstfarscript.so" % self.openfst_dirname,
            "%s/src/script/.libs/libfstscript.so" % self.openfst_dirname,
            "%s/src/lib/.libs/libfst.so" % self.openfst_dirname,
        ]

    @property
    def output_dir(self):
        return "%s/openfst_python" % self.build_lib

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.download_dir = None

    def finalize_options(self):
        build_ext.finalize_options(self)
        self.set_undefined_options("build", ("download_dir", "download_dir"))

    def openfst_download(self):
        if os.path.exists(self.openfst_dirname):
            return

        if not os.path.isdir(self.build_temp):
            os.makedirs(self.build_temp)

        if not os.path.exists(self.openfst_filename):
            print("downloading from %s" % self.openfst_url)
            r = requests.get(self.openfst_url, verify=False, stream=True)
            r.raw.decode_content = True
            with open(self.openfst_filename, "wb") as f:
                shutil.copyfileobj(r.raw, f)

    def openfst_extract(self):
        if not os.path.exists(self.openfst_dirname):
            extract_cmd = ["tar", "xzf", self.openfst_filename, "-C", self.build_temp]
            subprocess.check_call(extract_cmd)

    def openfst_configure_and_make(self):
        if not os.path.exists(self.openfst_main_lib):
            old_dir = os.getcwd()
            os.chdir(self.openfst_dirname)
            if os.path.exists("Makefile"):
                subprocess.check_call(["make", "distclean"])
            subprocess.check_call(["aclocal"])
            subprocess.check_call(["autoconf", "-f"])
            configure_cmd = [
                "./configure",
                "--enable-compact-fsts",
                "--enable-compress",
                "--enable-const-fsts",
                "--enable-far",
                "--enable-linear-fsts",
                "--enable-lookahead-fsts",
                "--enable-python",
                "--enable-special",
            ]
            subprocess.check_call(configure_cmd)
            subprocess.check_call(["make", "-j8"])
            os.chdir(old_dir)

    def openfst_copy_libraries(self, ext):
        main_lib_output_path = os.path.join(self.build_lib, ext._file_name)
        copy(self.openfst_main_lib, main_lib_output_path)
        for src in self.openfst_deps_libs:
            dst = "%s/%s" % (self.output_dir, get_filename_with_sha256(src))
            copy(src, dst)

    def openfst_fix_libraries(self):
        def patchelf_needed(so_filename):
            patchelf_cmd = ("patchelf", "--print-needed", so_filename)
            output = subprocess.check_output(patchelf_cmd)
            if isinstance(output, bytes):
                return output.decode("utf-8").split()
            else:
                return output.split()

        def patchelf_replace(so_filename, oldso, newso):
            patchelf_cmd = ("patchelf", "--replace-needed", oldso, newso, so_filename)
            subprocess.check_call(patchelf_cmd)

        def patchelf_remove(so_filename, oldso):
            patchelf_cmd = ("patchelf", "--remove-needed", oldso, so_filename)
            subprocess.check_call(patchelf_cmd)

        def patchelf_rpath_origin(so_filename):
            patchelf_cmd = ("patchelf", "--set-rpath", "$ORIGIN", so_filename)
            subprocess.check_call(patchelf_cmd)

        def patchelf_replace_all(so_filename, so_map):
            for oldso in patchelf_needed(so_filename):
                oldso_noversion = re.sub(r"^(.*\.so)\.[0-9]+$", r"\1", oldso)
                if oldso_noversion in so_map:
                    newso = so_map[oldso_noversion]
                    patchelf_replace(so_filename, oldso, newso)

        def patchelf_remove_libpython(so_filename):
            for oldso in patchelf_needed(so_filename):
                if re.match(r"^libpython.*$", oldso) is not None:
                    patchelf_remove(so_filename, oldso)

        somap = {
            os.path.basename(dep): get_filename_with_sha256(dep)
            for dep in self.openfst_deps_libs
        }

        for sofile in os.listdir(self.output_dir):
            if re.match(r"^.*(\.so).*$", sofile) is not None:
                sofile = os.path.join(self.output_dir, sofile)
                patchelf_replace_all(sofile, somap)
                patchelf_remove_libpython(sofile)
                patchelf_rpath_origin(sofile)

    def run(self):
        self.openfst_download()
        self.openfst_extract()
        self.openfst_configure_and_make()
        self.openfst_copy_libraries(self.extensions[0])
        self.openfst_fix_libraries()

        cmd = self.get_finalized_command("build_py").build_lib
        self.write_stub(cmd, self.extensions[0])


with open(os.path.join(os.path.dirname(__file__), "README.md"), "r") as fh:
    long_description = fh.read()

setup(
    name="openfst_python",
    version=OPENFST_VERSION,
    description="Stand-alone OpenFST bindings for Python",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/jpuigcerver/openfst-python",
    author="Joan Puigcerver",
    author_email="joapuipe@gmail.com",
    license="MIT",
    packages=find_packages(),
    ext_modules=[OpenFstExtension()],
    cmdclass=dict(build=OpenFstBuild, build_ext=OpenFstBuildExt),
    classifiers=[
        "Development Status :: 4 - Beta",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
        "Intended Audience :: Science/Research",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Topic :: Scientific/Engineering",
        "Topic :: Software Development",
        "Topic :: Software Development :: Libraries",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    setup_requires=["Cython", "requests"],
    zip_safe=False,
)