From 56c309d96ea8146c99efffd8509bb952c543ed8e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 5 May 2024 21:12:42 +0200 Subject: [PATCH] made sure that github groups the scraped parts nicely --- .github/workflows/update-data.yml | 4 ++++ data/external/main.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 6fe446c48..6e47eccb6 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -41,6 +41,10 @@ jobs: CONNECTUM_OAUTH_CLIENT_ID: ${{ secrets.CONECTUM_OAUTH_CLIENT_ID }} CONNECTUM_OAUTH_CLIENT_SECRET: ${{ secrets.CONECTUM_OAUTH_CLIENT_SECRET }} working-directory: data/external + - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious + run: ls -lah data/external/results + - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious + run: du -sh data/external/results/* - name: Fix linting errors continue-on-error: true uses: pre-commit/action@v3.0.1 diff --git a/data/external/main.py b/data/external/main.py index 717dd12dc..ca8803760 100644 --- a/data/external/main.py +++ b/data/external/main.py @@ -36,21 +36,29 @@ def ensure_is_empty(directory: Path): setup_logging(level=logging.INFO) ensure_is_empty(CACHE_PATH) + print("::group::downloading TUMonline information") tumonline.scrape_buildings() tumonline.scrape_rooms() tumonline.scrape_usages() tumonline.scrape_orgs(lang="de") tumonline.scrape_orgs(lang="en") + print("::endgroup::") + print("::group::downloading NAT information") os.makedirs(CACHE_PATH / "nat", exist_ok=True) nat.scrape_buildings() nat.scrape_rooms() + print("::endgroup::") + print("::group::downloading public_transport information") + os.makedirs(CACHE_PATH / "public_transport", exist_ok=True) + public_transport.scrape_stations() + print("::endgroup::") + + print("::group::downloading MyTUM information") os.makedirs(CACHE_PATH / "maps" / "roomfinder", exist_ok=True) os.makedirs(CACHE_PATH / "maps" / "roomfinder" / "kmz", exist_ok=True) roomfinder.scrape_buildings() roomfinder.scrape_rooms() roomfinder.scrape_maps() - - os.makedirs(CACHE_PATH / "public_transport", exist_ok=True) - public_transport.scrape_stations() + print("::endgroup::")