made sure that github groups the scraped parts nicely

.github/workflows/update-data.yml

@@ -41,6 +41,10 @@ jobs:
           CONNECTUM_OAUTH_CLIENT_ID: ${{ secrets.CONECTUM_OAUTH_CLIENT_ID }}
           CONNECTUM_OAUTH_CLIENT_SECRET: ${{ secrets.CONECTUM_OAUTH_CLIENT_SECRET }}
         working-directory: data/external
+      - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious
+        run: ls -lah data/external/results
+      - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious
+        run: du -sh data/external/results/*
       - name: Fix linting errors
         continue-on-error: true
         uses: pre-commit/[email protected]

data/external/main.py

@@ -36,21 +36,29 @@ def ensure_is_empty(directory: Path):
     setup_logging(level=logging.INFO)
     ensure_is_empty(CACHE_PATH)
 
+    print("::group::downloading TUMonline information")
     tumonline.scrape_buildings()
     tumonline.scrape_rooms()
     tumonline.scrape_usages()
     tumonline.scrape_orgs(lang="de")
     tumonline.scrape_orgs(lang="en")
+    print("::endgroup::")
 
+    print("::group::downloading NAT information")
     os.makedirs(CACHE_PATH / "nat", exist_ok=True)
     nat.scrape_buildings()
     nat.scrape_rooms()
+    print("::endgroup::")
 
+    print("::group::downloading public_transport information")
+    os.makedirs(CACHE_PATH / "public_transport", exist_ok=True)
+    public_transport.scrape_stations()
+    print("::endgroup::")
+
+    print("::group::downloading MyTUM information")
     os.makedirs(CACHE_PATH / "maps" / "roomfinder", exist_ok=True)
     os.makedirs(CACHE_PATH / "maps" / "roomfinder" / "kmz", exist_ok=True)
     roomfinder.scrape_buildings()
     roomfinder.scrape_rooms()
     roomfinder.scrape_maps()
-
-    os.makedirs(CACHE_PATH / "public_transport", exist_ok=True)
-    public_transport.scrape_stations()
+    print("::endgroup::")