From e93fb13aa6625570c2b60e1688da19f47a2de409 Mon Sep 17 00:00:00 2001 From: gmweaver Date: Wed, 5 Jun 2024 20:35:49 -0700 Subject: [PATCH] add deploy workflow --- .github/workflows/deploy.yml | 111 +++++++++++++++++++++++++++++++++++ .github/workflows/wheels.yml | 2 +- README.md | 29 +-------- libpypostal/parser.py | 2 +- src/pyparser.c | 83 ++++++++++++++++---------- 5 files changed, 167 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..d2acca1 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,111 @@ +name: Build and deploy to PyPI + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + release: + types: + - published + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-13, macos-latest] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.18.1 + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_SKIP: cp36-* cp37-* *musllinux* + CIBW_BEFORE_ALL_LINUX: > + yum install -y autoconf automake curl libtool pkgconfig sudo && + git clone https://github.com/openvenues/libpostal && + cd libpostal && + git checkout tags/v1.1 && + ./bootstrap.sh && + ./configure --disable-data-download --disable-sse2 && + make -j4 && + sudo make install && + sudo ldconfig && + pkg-config --cflags libpostal && + pkg-config --libs libpostal && + pkg-config --cflags --libs libpostal + CIBW_BEFORE_ALL_MACOS: > + brew install autoconf automake curl libtool pkg-config && + git clone https://github.com/openvenues/libpostal && + cd libpostal && + git checkout tags/v1.1 && + ./bootstrap.sh && + ./configure --disable-data-download --disable-sse2 && + make -j4 && + sudo make install && + pkg-config --cflags libpostal && + pkg-config --libs libpostal && + pkg-config --cflags --libs libpostal + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./dist/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + + - name: Build sdist + run: pipx run build --sdist + env: + CIBW_BEFORE_ALL: > + yum install -y autoconf automake curl libtool pkgconfig sudo && + git clone https://github.com/openvenues/libpostal && + cd libpostal && + git checkout tags/v1.1 && + ./bootstrap.sh && + ./configure --disable-data-download --disable-sse2 && + make -j4 && + sudo make install && + sudo ldconfig && + pkg-config --cflags libpostal && + pkg-config --libs libpostal && + pkg-config --cflags --libs libpostal + + - uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + + + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v4 + with: + pattern: cibw-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c3322cc..4696883 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,4 +1,4 @@ -name: Build +name: Build wheels on: [push, pull_request] diff --git a/README.md b/README.md index f551d58..b8e066d 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,11 @@ -# pylibpostal +# libpypostal Python wrapper for open-source libpostal project. Custom libary built internally due to lack of continued support for current Python wrapper libraries. ## Usage -### Install libpostal C library - -By default, libpostal will be installed when the Python package is installed, but without the data. - -The commands run to install are below. - -``` -git clone https://github.com/openvenues/libpostal \ - && cd libpostal \ - && ./bootstrap.sh \ - && ./configure --datadir=/tmp/libpostal_data_files --disable-data-download --disable-sse2 \ - && make -j4 \ - && make install \ - && ldconfig -``` - -- `--disable-data-download` disables downloading data when installing. -- `--disable-sse2` required for Mac M1. -- `ldconfig` only needed for linux. - -See https://github.com/openvenues/libpostal?tab=readme-ov-file#installation-maclinux for more details. - ### Downloading libpostal data ``` libpostal_data download all ``` - -## Contributing -To test the project, run `poetry test`. Test files may live together with the code or in a separate -directory, but in order for them to be discovered, they should end with `_test.py` -(e.g. `pylibpostal/something_test.py` or `pylibpostal_test/something_test.py`). diff --git a/libpypostal/parser.py b/libpypostal/parser.py index 2596a43..82fd9e6 100644 --- a/libpypostal/parser.py +++ b/libpypostal/parser.py @@ -29,7 +29,7 @@ class LibpostalAddressComponent(str, Enum): def _parse_address( - address: str, language: str, country_code: str + address: str, language: Optional[str] = None, country_code: Optional[str] = None ) -> List[Tuple[str, str]]: from libpypostal import _parser # type: ignore # pylint: disable=no-name-in-module,import-outside-toplevel diff --git a/src/pyparser.c b/src/pyparser.c index 786b827..b9b139d 100644 --- a/src/pyparser.c +++ b/src/pyparser.c @@ -3,55 +3,55 @@ #include "pyutils.h" -struct module_state { +struct module_state +{ PyObject *error; }; #define GETSTATE(m) ((struct module_state *)PyModule_GetState(m)) static PyObject *py_parse_address(PyObject *self, PyObject *args, - PyObject *keywords) { + PyObject *keywords) +{ PyObject *arg_input; PyObject *arg_language = Py_None; PyObject *arg_country = Py_None; PyObject *result = NULL; - char *datadir = getenv("LIBPOSTAL_DATA_DIR"); - - if ((datadir != NULL) && (!libpostal_setup_datadir(datadir) || - !libpostal_setup_parser_datadir(datadir)) || - (!libpostal_setup() || !libpostal_setup_parser())) { - PyErr_SetString(PyExc_TypeError, "Error loading libpostal data"); - } - static char *kwlist[] = {"address", "language", "country", NULL}; if (!PyArg_ParseTupleAndKeywords(args, keywords, "O|OO:pyparser", kwlist, - &arg_input, &arg_language, &arg_country)) { + &arg_input, &arg_language, &arg_country)) + { return 0; } char *input = PyObject_to_string(arg_input); - if (input == NULL) { + if (input == NULL) + { return NULL; } char *language = NULL; - if (arg_language != Py_None) { + if (arg_language != Py_None) + { language = PyObject_to_string(arg_language); - if (language == NULL) { + if (language == NULL) + { goto exit_free_input; } } char *country = NULL; - if (arg_country != Py_None) { + if (arg_country != Py_None) + { country = PyObject_to_string(arg_language); - if (country == NULL) { + if (country == NULL) + { goto exit_free_language; } } @@ -63,35 +63,41 @@ static PyObject *py_parse_address(PyObject *self, PyObject *args, libpostal_address_parser_response_t *parsed = libpostal_parse_address(input, options); - if (parsed == NULL) { + if (parsed == NULL) + { goto exit_free_country; } result = PyList_New((Py_ssize_t)parsed->num_components); - if (!result) { + if (!result) + { goto exit_destroy_response; } - for (int i = 0; i < parsed->num_components; i++) { + for (int i = 0; i < parsed->num_components; i++) + { char *component = parsed->components[i]; char *label = parsed->labels[i]; PyObject *component_unicode = PyUnicode_DecodeUTF8( (const char *)component, strlen(component), "strict"); - if (component_unicode == NULL) { + if (component_unicode == NULL) + { Py_DECREF(result); goto exit_destroy_response; } PyObject *label_unicode = PyUnicode_DecodeUTF8((const char *)label, strlen(label), "strict"); - if (label_unicode == NULL) { + if (label_unicode == NULL) + { Py_DECREF(component_unicode); Py_DECREF(result); goto exit_destroy_response; } PyObject *tuple = Py_BuildValue("(OO)", component_unicode, label_unicode); - if (tuple == NULL) { + if (tuple == NULL) + { Py_DECREF(component_unicode); Py_DECREF(label_unicode); goto exit_destroy_response; @@ -107,15 +113,18 @@ static PyObject *py_parse_address(PyObject *self, PyObject *args, exit_destroy_response: libpostal_address_parser_response_destroy(parsed); exit_free_country: - if (country != NULL) { + if (country != NULL) + { free(country); } exit_free_language: - if (language != NULL) { + if (language != NULL) + { free(language); } exit_free_input: - if (input != NULL) { + if (input != NULL) + { free(input); } return result; @@ -127,12 +136,14 @@ static PyMethodDef parser_methods[] = { {NULL, NULL}, }; -static int parser_traverse(PyObject *m, visitproc visit, void *arg) { +static int parser_traverse(PyObject *m, visitproc visit, void *arg) +{ Py_VISIT(GETSTATE(m)->error); return 0; } -static int parser_clear(PyObject *m) { +static int parser_clear(PyObject *m) +{ Py_CLEAR(GETSTATE(m)->error); libpostal_teardown(); libpostal_teardown_parser(); @@ -151,19 +162,31 @@ static struct PyModuleDef module_def = {PyModuleDef_HEAD_INIT, #define INITERROR return NULL -PyObject *PyInit__parser(void) { +PyObject *PyInit__parser(void) +{ PyObject *module = PyModule_Create(&module_def); - if (module == NULL) { + if (module == NULL) + { INITERROR; } struct module_state *st = GETSTATE(module); st->error = PyErr_NewException("_parser.Error", NULL, NULL); - if (st->error == NULL) { + if (st->error == NULL) + { Py_DECREF(module); INITERROR; } + char *datadir = getenv("LIBPOSTAL_DATA_DIR"); + + if ((datadir != NULL) && (!libpostal_setup_datadir(datadir) || + !libpostal_setup_parser_datadir(datadir)) || + (!libpostal_setup() || !libpostal_setup_parser())) + { + PyErr_SetString(PyExc_TypeError, "Error loading libpostal data"); + } + return module; }