diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index f71d4de..53697b6 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -1,4 +1,4 @@ -name: Build CI Img +name: ๐Ÿ”ง Build CI Img on: # Push includes PR merge @@ -12,7 +12,7 @@ on: workflow_dispatch: jobs: - backend-ci-build: + ci-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.0.1 with: build_target: ci diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72b2b3f..9f9ab23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build +name: ๐Ÿ”ง Build on: push: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 10d2bda..c7ba2af 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,4 @@ -name: Publish Docs +name: ๐Ÿ“– Publish Docs on: push: diff --git a/.github/workflows/pr_label.yml b/.github/workflows/pr_label.yml index c952171..1e5a389 100644 --- a/.github/workflows/pr_label.yml +++ b/.github/workflows/pr_label.yml @@ -1,4 +1,4 @@ -name: PR Label +name: ๐Ÿท๏ธ PR Label on: pull_request_target: @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/labeler@v3 + - uses: actions/labeler@v4 # Uses .github/labeler.yml definitions with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c560a2c..e6b4e90 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Publish to PyPi.org +name: ๐Ÿšฉ Publish to PyPi.org on: release: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 864975f..98d9427 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,4 +1,4 @@ -name: PyTest +name: ๐Ÿงช PyTest on: # Run tests on all pushed branches diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml new file mode 100644 index 0000000..6e4762e --- /dev/null +++ b/.github/workflows/wiki.yml @@ -0,0 +1,15 @@ +name: ๐Ÿ“– Publish Docs to Wiki + +on: + push: + paths: + - docs/** + branches: [main] + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + publish-docs-to-wiki: + uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.0.1 + with: + homepage_path: "index.md" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d9743a..4119a63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,14 +8,14 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.0.292" + rev: "v0.1.1" hooks: - id: ruff args: [--exit-non-zero-on-fix] diff --git a/README.md b/README.md index 5b1931f..3bd1196 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ --- -๐Ÿ•ฎ **Documentation**: https://hotosm.github.io/osm-rawdata/ +๐Ÿ“– **Documentation**: https://hotosm.github.io/osm-rawdata/ ๐Ÿ–ฅ๏ธ **Source Code**: https://github.com/hotosm/osm-rawdata diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md new file mode 100644 index 0000000..8be9605 --- /dev/null +++ b/docs/_Sidebar.md @@ -0,0 +1,17 @@ +# [Home](https://github.com/hotosm/osm-fieldwork/wiki) + +[About](https://github.com/hotosm/osm-fieldwork/wiki/about) + +[Postgres](https://github.com/hotosm/osm-fieldwork/wiki/postgres) + +[YAML](https://github.com/hotosm/osm-fieldwork/wiki/yaml) + +[JSON](https://github.com/hotosm/osm-fieldwork/wiki/json) + +[Importer](https://github.com/hotosm/osm-fieldwork/wiki/importer) + +[Overture](https://github.com/hotosm/osm-fieldwork/wiki/overture) + +[Geofabrik](https://github.com/hotosm/osm-fieldwork/wiki/geofabrik) + +[API](https://github.com/hotosm/osm-fieldwork/wiki/api) diff --git a/docs/about.md b/docs/about.md index bc4d84e..74f8189 100644 --- a/docs/about.md +++ b/docs/about.md @@ -1,6 +1,6 @@ # osm-rawdata -๐Ÿ•ฎ [Documentation](https://hotosm.github.io/osm-rawdata/) +๐Ÿ“– [Documentation](https://hotosm.github.io/osm-rawdata/) These is a module to work with [OpenStreetMap](https://www.openstreetmap.org) data using postgres and diff --git a/docs/geofabrik.md b/docs/geofabrik.md index 622ad65..1e3a7c8 100644 --- a/docs/geofabrik.md +++ b/docs/geofabrik.md @@ -1,10 +1,10 @@ # Geofabrik This is a simple utility to download country data files from -[GeoFabrik](https://download.geofabrik.de/). +[GeoFabrik](https://download.geofabrik.de/). - options: - --help(-h) show this help message and exit - --verbose(-v) verbose output - --file(-f) FILE The country or US state to download - --list(-l) List all files on GeoFabrik + options: + --help(-h) show this help message and exit + --verbose(-v) verbose output + --file(-f) FILE The country or US state to download + --list(-l) List all files on GeoFabrik diff --git a/docs/overture.md b/docs/overture.md index e6dead0..e3f6712 100644 --- a/docs/overture.md +++ b/docs/overture.md @@ -1,6 +1,6 @@ # Overture Map Data -The Overture Foundation (https://www.overturemaps.org) has been +The Overture Foundation () has been recently formed to build a competitor to Google Maps. The plan is to use OpenStreetMap (OSM) data as a baselayer, and layer other datasets on top. The currently available data (July 2023) has 13 different @@ -40,28 +40,27 @@ less columns in it, and each data type had a schema oriented towards that data type. The new schema (Oct 2023) is larger, but all the data types are supported in the same schema. -The schema used in the Overture data files is [documented here]( -https://docs.overturemaps.org/reference). This document is just a +The schema used in the Overture data files is [documented here](https://docs.overturemaps.org/reference). This document is just a summary with some implementation details. ### Buildings The current list of buildings datasets is: -* Austin Building Footprints Year 2013 2D Buildings -* Boston BPDA 3D Buildings -* City of Cambridge, MA Open Data 3D Buildings -* Denver Regional Council of Governments 2D Buildings -* Esri Buildings | Austin Building Footprints Year 2013 2D Buildings -* Esri Buildings | Denver Regional Council of Governments 2D Buildings -* Esri Community Maps -* Miami-Dade County Open Data 3D Buildings -* OpenStreetMap -* Microsoft ML Buildings -* NYC Open Data 3D Buildings -* Portland Building Footprint 2D Buildings -* USGS Lidar -* Washington DC Open Data 3D Buildings +- Austin Building Footprints Year 2013 2D Buildings +- Boston BPDA 3D Buildings +- City of Cambridge, MA Open Data 3D Buildings +- Denver Regional Council of Governments 2D Buildings +- Esri Buildings | Austin Building Footprints Year 2013 2D Buildings +- Esri Buildings | Denver Regional Council of Governments 2D Buildings +- Esri Community Maps +- Miami-Dade County Open Data 3D Buildings +- OpenStreetMap +- Microsoft ML Buildings +- NYC Open Data 3D Buildings +- Portland Building Footprint 2D Buildings +- USGS Lidar +- Washington DC Open Data 3D Buildings Since the Microsoft ML Buildings and the OpenStreetMap data is available elsewhere, and is more up-to-date for global coverage, all @@ -78,30 +77,30 @@ accurate. ### Places -The *places* data are POIs of places. This appears to be for +The _places_ data are POIs of places. This appears to be for amenities, and contains tags related to that OSM category. This dataset is from Meta, and the data appears derived from Facebook. The columns that are of interest to OSM are: -* freeform - The address of the amenity, although the format is not +- freeform - The address of the amenity, although the format is not consistent -* socials - An array of social media links for this amenity. -* phone - The phone number if it has one -* websites - The website URL if it has one -* value - The name of the amenity if known +- socials - An array of social media links for this amenity. +- phone - The phone number if it has one +- websites - The website URL if it has one +- value - The name of the amenity if known ### Highways -In the current highway *segment* data files, the only source is +In the current highway _segment_ data files, the only source is OSM. In that cases it's better to use uptodate OSM data. It'll be interesting to see if Overture imports the publically available highway datasets from the USGS, or some state governments. That would be very useful. -The Overture *segments* data files are equivalent to an OSM way, with +The Overture _segments_ data files are equivalent to an OSM way, with tags specific to that highway linestring. There are separate data -files for *connections*, that are equivalant to an OSM relation. +files for _connections_, that are equivalant to an OSM relation. ### Admin Boundaries @@ -115,10 +114,10 @@ reason to care about these files. The names column can have 4 variations on the name. Each may also have a language value as well. -* common -* official -* alternate -* short +- common +- official +- alternate +- short Each of these can have multiple values, each of which consists of a value and the language. @@ -126,10 +125,10 @@ value and the language. ## sources The sources column is an array of with two entries. The first entry is -the name of the dataset, and where it exists, a *recordID* to +the name of the dataset, and where it exists, a _recordID_ to reference the source dataset. For OSM data, the recordID has 3 -sub-fields. The first character is the type, *w* (way), *n* (node), or -*l* (line). The second is the OSM ID, and the third with a *v* is the +sub-fields. The first character is the type, _w_ (way), _n_ (node), or +_l_ (line). The second is the OSM ID, and the third with a _v_ is the version of the feature in OSM. -For example: *w***123456**v2 is a way with ID 123456 and is version 2. +For example: \*w**\*123456**v2 is a way with ID 123456 and is version 2. diff --git a/docs/postgres.md b/docs/postgres.md index 63e7c45..666d13d 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -6,12 +6,12 @@ from a local postgres data, or the remote Underpass one. A boundary polygon is used to define the area to be covered in the extract. Optionally a data file can be used. - options: - --help(-h) show this help message and exit - --verbose(-v) verbose output - --uri(-u) URI Database URI - --boundary(-b) BOUNDARY Boundary polygon to limit the data size - --sql(-s) SQL Custom SQL query to execute against the database - --all(-a) ALL All the geometry or just centroids - --config(-c) CONFIG The config file for the query (json or yaml) - --outfile(-o) OUTFILE The output file + options: + --help(-h) show this help message and exit + --verbose(-v) verbose output + --uri(-u) URI Database URI + --boundary(-b) BOUNDARY Boundary polygon to limit the data size + --sql(-s) SQL Custom SQL query to execute against the database + --all(-a) ALL All the geometry or just centroids + --config(-c) CONFIG The config file for the query (json or yaml) + --outfile(-o) OUTFILE The output file diff --git a/osm_rawdata/postgres.py b/osm_rawdata/postgres.py index e48b207..b12af62 100755 --- a/osm_rawdata/postgres.py +++ b/osm_rawdata/postgres.py @@ -370,7 +370,7 @@ def queryLocal( return FeatureCollection(features) # If there is no config file, don't modify the results - if len(self.qc.config['where']['ways_poly']) == 0 and len(self.qc.config['where']['nodes']) == 0: + if len(self.qc.config["where"]["ways_poly"]) == 0 and len(self.qc.config["where"]["nodes"]) == 0: return result for item in result: