From d2d2bdbc3d15ef9260686249ef709e617a86d86d Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 5 May 2024 23:34:46 +0200 Subject: [PATCH] replaced the SED calls of the workflow into actual python code to get around escaping errors --- .github/workflows/update-data.yml | 9 +-------- data/README.md | 2 +- data/external/scrapers/roomfinder.py | 6 ++++++ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 5e24c9301..3f8be801c 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -44,14 +44,7 @@ jobs: working-directory: data/external - name: Download mytum data continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious - run: | - PYTHONPATH=$PYTHONPATH:.. python3 scrapers/roomfinder.py - sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/buildings_roomfinder.json - sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/buildings_roomfinder.json - sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/maps_roomfinder.json - sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/maps_roomfinder.json - sed -i 's/Bestelmeyer S\u00fcd/Zentralgeb\u00e4ude 2/g' results/rooms_roomfinder.json - sed -i 's/Bestelmeyer Nord/Zentralgeb\u00e4ude 7/g' results/rooms_roomfinder.json + run: PYTHONPATH=$PYTHONPATH:.. python3 scrapers/roomfinder.py env: TQDM_MININTERVAL: 100 working-directory: data/external diff --git a/data/README.md b/data/README.md index 9f3ecb4a3..48b5c1167 100644 --- a/data/README.md +++ b/data/README.md @@ -10,7 +10,7 @@ The code to retrieve external data, as well as externally retrieved data is loca > [!WARNING] > A lot of this code is more a work in progress than finished. Especially features such as POIs, custom maps or other data types such as events are drafted but not yet fully implemented. -> +> > New external data might break the scripts from time to time, as either rooms or buildings are removed, the external data has errors or we make assumptions here that turn out to be wrong. ## Getting started diff --git a/data/external/scrapers/roomfinder.py b/data/external/scrapers/roomfinder.py index 998fe1eed..edd3ca425 100644 --- a/data/external/scrapers/roomfinder.py +++ b/data/external/scrapers/roomfinder.py @@ -35,6 +35,8 @@ def _sanitise_building(building: dict): building["lat"], building["lon"] = 48.14903, 11.56735 else: building["lat"], building["lon"] = _utm_to_latlon(easting, northing, zone_number) + building["b_name"].replace("Bestelmeyer Nord", "Zentralgebäude 7") # TUM changed names in denazification initative + building["b_name"].replace("Bestelmeyer Süd", "Zentralgebäude 2") # TODO: Remove if MyTUM does fix this.. def scrape_buildings() -> None: @@ -87,6 +89,8 @@ def _sanitise_room(room: dict) -> dict: easting=room.pop("utm_easting"), northing=room.pop("utm_northing"), ) + room["b_name"].replace("Bestelmeyer Nord", "Zentralgebäude 7") # TUM changed names in denazification initative + room["b_name"].replace("Bestelmeyer Süd", "Zentralgebäude 2") # TODO: Remove if MyTUM does fix this.. return room @@ -208,6 +212,8 @@ def _download_maps(used_maps): _download_file(url, filepath) convert_to_webp(filepath) + _map[2].replace("Bestelmeyer Nord", "Zentralgebäude 7") # TUM changed names in denazification initative + _map[2].replace("Bestelmeyer Süd", "Zentralgebäude 2") # TODO: Remove if MyTUM does fix this.. map_data = { "scale": _map[0], "id": _map[1],