diff --git a/.editorconfig b/.editorconfig index f5444d81a74..572b73bd2ed 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,7 @@ insert_final_newline = true indent_style = space indent_size = 2 trim_trailing_whitespace = false + +[*.yml] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000000..d59f8e1ec61 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,136 @@ +name: Build Wheels for PyPI +on: + workflow_dispatch: + +jobs: + build_wheels: + strategy: + fail-fast: false + matrix: + os: [ + { + name: "Ubuntu 22.04", + family: "linux", + runner: "ubuntu-22.04", + archs: "x86_64", + }, + ## Aarch64 is disabled for now: GitHub is committing to EOY + ## for free aarch64 runners for open-source projects and + ## emulation times out: + ## https://github.com/orgs/community/discussions/19197#discussioncomment-10550689 + # { + # name: "Ubuntu 22.04", + # family: "linux", + # runner: "ubuntu-22.04", + # archs: "aarch64", + # }, + { + name: "macOS 13", + family: "macos", + runner: "macos-13", + archs: "x86_64", + }, + { + name: "macOS 14", + family: "macos", + runner: "macos-14", + archs: "arm64", + }, + ## Windows is disabled because of an issue with compiling FFI as + ## under MinGW in the GitHub Actions environment (SHELL variable has + ## whitespace.) + # { + # name: "Windows Server 2019", + # family: "windows", + # runner: "windows-2019", + # archs: "AMD64", + # }, + ] + name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }} + runs-on: ${{ matrix.os.runner }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + - if: ${{ matrix.os.family == 'linux' }} + name: "[Linux] Set up QEMU" + uses: docker/setup-qemu-action@v3 + - uses: actions/setup-python@v5 + - name: Get Boost Source + shell: bash + run: | + mkdir -p boost + curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost + - name: Get FFI + shell: bash + run: | + mkdir -p ffi + curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi + ## Software installed by default in GitHub Action Runner VMs: + ## https://github.com/actions/runner-images + - if: ${{ matrix.os.family == 'macos' }} + name: "[macOS] Flex/Bison" + run: | + brew install flex bison + echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV + - if: ${{ matrix.os.family == 'windows' }} + name: "[Windows] Flex/Bison" + run: | + choco install winflexbison3 + - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }} + name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)" + uses: actions/setup-python@v5 + with: + python-version: 3.8 + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.1 + env: + # * APIs not supported by PyPy + # * Musllinux disabled because it increases build time from 48m to ~3h + CIBW_SKIP: > + pp* + *musllinux* + CIBW_ARCHS: ${{ matrix.os.archs }} + CIBW_BUILD_VERBOSITY: "1" + # manylinux2014 (default) does not have a modern enough C++ compiler for Yosys + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh + CIBW_ENVIRONMENT: > + CXXFLAGS=-I./boost/pfx/include + LINKFLAGS=-L./boost/pfx/lib + PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig + makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a' + CIBW_ENVIRONMENT_MACOS: > + CXXFLAGS=-I./boost/pfx/include + LINKFLAGS=-L./boost/pfx/lib + PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig + MACOSX_DEPLOYMENT_TARGET=11 + makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang' + CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh + CIBW_TEST_COMMAND: python3 -c "from pyosys import libyosys as ys;d=ys.Design();ys.run_pass('help', d)" + - uses: actions/upload-artifact@v4 + with: + name: python-wheels-${{ matrix.os.runner }} + path: ./wheelhouse/*.whl + upload_wheels: + name: Upload Wheels + runs-on: ubuntu-latest + needs: build_wheels + steps: + - uses: actions/download-artifact@v4 + with: + path: "." + pattern: python-wheels-* + merge-multiple: true + - run: | + ls + mkdir -p ./dist + mv *.whl ./dist + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + repository-url: ${{ vars.PYPI_INDEX || 'https://upload.pypi.org/legacy/' }} diff --git a/.github/workflows/wheels/_run_cibw_linux.py b/.github/workflows/wheels/_run_cibw_linux.py new file mode 100644 index 00000000000..894470a5a8d --- /dev/null +++ b/.github/workflows/wheels/_run_cibw_linux.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024 Efabless Corporation +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" +This runs the cibuildwheel step from the wheels workflow locally. +""" + +import os +import yaml +import platform +import subprocess + +__dir__ = os.path.dirname(os.path.abspath(__file__)) + + +workflow = yaml.safe_load(open(os.path.join(os.path.dirname(__dir__), "wheels.yml"))) + +env = os.environ.copy() + +steps = workflow["jobs"]["build_wheels"]["steps"] +cibw_step = None +for step in steps: + if (step.get("uses") or "").startswith("pypa/cibuildwheel"): + cibw_step = step + break + +for key, value in cibw_step["env"].items(): + if key.endswith("WIN") or key.endswith("MAC"): + continue + env[key] = value + +env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine() +subprocess.check_call(["cibuildwheel"], env=env) diff --git a/.github/workflows/wheels/cibw_before_all.sh b/.github/workflows/wheels/cibw_before_all.sh new file mode 100644 index 00000000000..fbb8dcad830 --- /dev/null +++ b/.github/workflows/wheels/cibw_before_all.sh @@ -0,0 +1,23 @@ +set -e +set -x + +# Build-time dependencies +## Linux Docker Images +if command -v yum &> /dev/null; then + yum install -y flex bison +fi + +if command -v apk &> /dev/null; then + apk add flex bison +fi + +## macOS/Windows -- installed in GitHub Action itself, not container + +# Build Static FFI (platform-dependent but not Python version dependent) +cd ffi +## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries +CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx +## Without this, SHELL has a space in its path which breaks the makefile +make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu) +## Forces static library to be used in all situations +sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc diff --git a/.github/workflows/wheels/cibw_before_build.sh b/.github/workflows/wheels/cibw_before_build.sh new file mode 100644 index 00000000000..018b0e0df96 --- /dev/null +++ b/.github/workflows/wheels/cibw_before_build.sh @@ -0,0 +1,34 @@ +set -e +set -x + +# Don't use objects from previous compiles on Windows/macOS +make clean + +# DEBUG: show python3 and python3-config outputs +if [ "$(uname)" != "Linux" ]; then + # https://github.com/pypa/cibuildwheel/issues/2021 + ln -s $(dirname $(readlink -f $(which python3)))/python3-config $(dirname $(which python3))/python3-config +fi +python3 --version +python3-config --includes + +# Build boost +cd ./boost +## Delete the artefacts from previous builds (if any) +rm -rf ./pfx +## Bootstrap bjam +./bootstrap.sh --prefix=./pfx +## Build Boost against current version of Python, only for +## static linkage (Boost is statically linked because system boost packages +## wildly vary in versions, including the libboost_python3 version) +./b2\ + -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\ + --prefix=./pfx\ + --with-filesystem\ + --with-system\ + --with-python\ + cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\ + cflags="$(python3-config --includes) -fPIC"\ + link=static\ + variant=release\ + install diff --git a/.gitignore b/.gitignore index e82343e8947..5c3d8f4e963 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,10 @@ __pycache__ /tests/unit/bintest/ /tests/unit/objtest/ /tests/ystests -/result \ No newline at end of file +/result +/dist +/*.egg-info +/build +/venv +/boost +/ffi diff --git a/Makefile b/Makefile index 92274a56d67..f4276b94d9b 100644 --- a/Makefile +++ b/Makefile @@ -742,7 +742,7 @@ $(PROGRAM_PREFIX)yosys$(EXE): $(OBJS) libyosys.so: $(filter-out kernel/driver.o,$(OBJS)) ifeq ($(OS), Darwin) - $(P) $(CXX) -o libyosys.so -shared -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) + $(P) $(CXX) -o libyosys.so -shared -undefined dynamic_lookup -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) else $(P) $(CXX) -o libyosys.so -shared -Wl,-soname,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) endif diff --git a/kernel/yosys.cc b/kernel/yosys.cc index efdc54b796c..510151479b6 100644 --- a/kernel/yosys.cc +++ b/kernel/yosys.cc @@ -555,10 +555,15 @@ void yosys_setup() #undef X #ifdef WITH_PYTHON - PyImport_AppendInittab((char*)"libyosys", INIT_MODULE); - Py_Initialize(); - PyRun_SimpleString("import sys"); - signal(SIGINT, SIG_DFL); + // With Python 3.12, calling PyImport_AppendInittab on an already + // initialized platform fails (such as when libyosys is imported + // from a Python interpreter) + if (!Py_IsInitialized()) { + PyImport_AppendInittab((char*)"libyosys", INIT_MODULE); + Py_Initialize(); + PyRun_SimpleString("import sys"); + signal(SIGINT, SIG_DFL); + } #endif Pass::init_register(); diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..22196597109 --- /dev/null +++ b/setup.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024 Efabless Corporation +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import os +import re +import shlex +import shutil +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext + +__dir__ = os.path.dirname(os.path.abspath(__file__)) + +yosys_version_rx = re.compile(r"YOSYS_VER\s*:=\s*([\w\-\+\.]+)") + +version = yosys_version_rx.search( + open(os.path.join(__dir__, "Makefile"), encoding="utf8").read() +)[1].replace( + "+", "." +) # Convert to patch version + + +class libyosys_so_ext(Extension): + def __init__( + self, + ) -> None: + super().__init__( + "libyosys.so", + [], + ) + self.args = [ + "ENABLE_PYOSYS=1", + # Wheel meant to be imported from interpreter + "ENABLE_PYTHON_CONFIG_EMBED=0", + # Would need to be installed separately by the user + "ENABLE_TCL=0", + # Would need to be installed separately by the user + "ENABLE_READLINE=0", + # Show compile commands + "PRETTY=0", + ] + + def custom_build(self, bext: build_ext): + bext.spawn( + ["make", f"-j{os.cpu_count() or 1}", self.name] + + shlex.split(os.getenv("makeFlags", "")) + + self.args + ) + build_path = os.path.dirname(os.path.dirname(bext.get_ext_fullpath(self.name))) + pyosys_path = os.path.join(build_path, "pyosys") + target = os.path.join(pyosys_path, os.path.basename(self.name)) + os.makedirs(pyosys_path, exist_ok=True) + shutil.copyfile(self.name, target) + + # I don't know how debug info is getting here. + bext.spawn(["strip", "-S", target]) + + +class custom_build_ext(build_ext): + def build_extension(self, ext) -> None: + if not hasattr(ext, "custom_build"): + return super().build_extension(ext) + return ext.custom_build(self) + + +setup( + name="pyosys", + packages=["pyosys"], + version=version, + description="Python access to libyosys", + long_description=open(os.path.join(__dir__, "README.md")).read(), + long_description_content_type="text/markdown", + install_requires=["wheel", "setuptools"], + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + ], + package_dir={"pyosys": "misc"}, + python_requires=">=3.8", + ext_modules=[libyosys_so_ext()], + cmdclass={ + "build_ext": custom_build_ext, + }, +)