From 1f2221869ad4f61525d7cd8afa6031db6ef7a767 Mon Sep 17 00:00:00 2001 From: ben_29 Date: Sat, 2 Mar 2024 21:51:39 +0800 Subject: [PATCH] fix: garmin cn to global, add name and source fields --- run_page/config.py | 1 + run_page/garmin_sync.py | 36 +++++++++++++++++++++++++++-- run_page/garmin_sync_cn_global.py | 30 +++++++++++++++++++++--- run_page/generator/__init__.py | 7 +++++- run_page/gpxtrackposter/track.py | 4 ++++ run_page/synced_data_file_logger.py | 19 ++++++++++++++- 6 files changed, 90 insertions(+), 7 deletions(-) diff --git a/run_page/config.py b/run_page/config.py index 2ed683340d0..daac91c752b 100644 --- a/run_page/config.py +++ b/run_page/config.py @@ -20,6 +20,7 @@ JSON_FILE = os.path.join(parent, "src", "static", "activities.json") SYNCED_FILE = os.path.join(parent, "imported.json") SYNCED_ACTIVITY_FILE = os.path.join(parent, "synced_activity.json") +NAME_MAPPING_FILE = os.path.join(FIT_FOLDER, "name_mapping.json") # TODO: Move into nike_sync NRC THINGS diff --git a/run_page/garmin_sync.py b/run_page/garmin_sync.py index 3951fa49891..c5bd04a7d9f 100644 --- a/run_page/garmin_sync.py +++ b/run_page/garmin_sync.py @@ -31,7 +31,7 @@ "MODERN_URL": "https://connectapi.garmin.com", "SIGNIN_URL": "https://sso.garmin.com/sso/signin", "UPLOAD_URL": "https://connectapi.garmin.com/upload-service/upload/", - "ACTIVITY_URL": "https://connectapi.garmin.com/activity-service/activity/{activity_id}", + "ACTIVITY_URL": "https://connectapi.garmin.com/activity-service/activity/{}", } GARMIN_CN_URL_DICT = { @@ -40,7 +40,7 @@ "MODERN_URL": "https://connectapi.garmin.cn", "SIGNIN_URL": "https://sso.garmin.cn/sso/signin", "UPLOAD_URL": "https://connectapi.garmin.cn/upload-service/upload/", - "ACTIVITY_URL": "https://connectapi.garmin.cn/activity-service/activity/{activity_id}", + "ACTIVITY_URL": "https://connectapi.garmin.cn/activity-service/activity/{}", } @@ -108,6 +108,19 @@ async def get_activities(self, start, limit): url = url + "&activityType=running" return await self.fetch_data(url) + async def get_activity(self, activity_id): + """ + Fetch activity detail + """ + url = self.activity_url.format(activity_id) + data = {} + try: + data = await self.fetch_data(url) + except Exception as e: + print(f"fetch error: {activity_id} {str(e)}") + # just pass for now + return data + async def download_activity(self, activity_id, file_type="gpx"): url = f"{self.modern_url}/download-service/export/{file_type}/activity/{activity_id}" if file_type == "fit": @@ -301,6 +314,25 @@ async def download_new_activities( return to_generate_garmin_ids +async def get_activities_name(secret_string, auth_domain, new_ids): + client = Garmin(secret_string, auth_domain) + print(f"{len(new_ids)} new activities detail to be fetch") + + start_time = time.time() + returns = await gather_with_concurrency( + 10, + [client.get_activity(id) for id in new_ids], + ) + print(f"Fetch finished. Elapsed {time.time()-start_time} seconds") + + await client.req.aclose() + names_mapping = {} + for i in returns: + if "activityId" in i: + names_mapping[i["activityId"]] = i["activityName"] + return names_mapping + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/run_page/garmin_sync_cn_global.py b/run_page/garmin_sync_cn_global.py index 8d4ab0cd58b..cf5f3831f0e 100644 --- a/run_page/garmin_sync_cn_global.py +++ b/run_page/garmin_sync_cn_global.py @@ -20,8 +20,17 @@ from config import FIT_FOLDER, GPX_FOLDER, JSON_FILE, SQL_FILE, config from garmin_device_adaptor import wrap_device_info from garmin_sync import Garmin, get_downloaded_ids -from garmin_sync import download_new_activities, gather_with_concurrency -from synced_data_file_logger import load_synced_activity_list, save_synced_activity_list +from garmin_sync import ( + download_new_activities, + gather_with_concurrency, + get_activities_name, +) +from synced_data_file_logger import ( + load_synced_activity_list, + save_synced_activity_list, + load_fit_name_mapping, + save_fit_name_mapping, +) from utils import make_activities_file if __name__ == "__main__": @@ -74,6 +83,21 @@ loop.run_until_complete(future) new_ids = future.result() + # Step 2: + # activity name not included in fit file + # manually fetch activity name, save to file for later use + old_name_mapping = load_fit_name_mapping() + new_ids = new_ids - old_name_mapping.keys() + if new_ids: + loop = asyncio.get_event_loop() + future = asyncio.ensure_future( + get_activities_name(secret_string_cn, auth_domain, new_ids) + ) + loop.run_until_complete(future) + names_mapping = future.result() + names_mapping.update(old_name_mapping) + save_fit_name_mapping(names_mapping) + to_upload_files = [] for i in new_ids: if os.path.exists(os.path.join(FIT_FOLDER, f"{i}.fit")): @@ -99,7 +123,7 @@ synced_activity.extend(new_ids) save_synced_activity_list(synced_activity) - # Step 2: + # Step 3: # Generate track from fit/gpx file make_activities_file(SQL_FILE, GPX_FOLDER, JSON_FILE, file_suffix="gpx") make_activities_file(SQL_FILE, FIT_FOLDER, JSON_FILE, file_suffix="fit") diff --git a/run_page/generator/__init__.py b/run_page/generator/__init__.py index 5aa7b29e0a6..381e82edc57 100644 --- a/run_page/generator/__init__.py +++ b/run_page/generator/__init__.py @@ -12,7 +12,7 @@ from .db import Activity, init_db, update_or_create_activity -from synced_data_file_logger import save_synced_data_file_list +from synced_data_file_logger import save_synced_data_file_list, load_fit_name_mapping IGNORE_BEFORE_SAVING = os.getenv("IGNORE_BEFORE_SAVING", False) @@ -84,8 +84,13 @@ def sync_from_data_dir(self, data_dir, file_suffix="gpx"): return synced_files = [] + if file_suffix == "fit": + name_mapping = load_fit_name_mapping() for t in tracks: + activity_id = t.file_names[0].split(".")[0] + if file_suffix == "fit" and activity_id in name_mapping: + t.name = name_mapping[activity_id] created = update_or_create_activity(self.session, t.to_namedtuple()) if created: sys.stdout.write("+") diff --git a/run_page/gpxtrackposter/track.py b/run_page/gpxtrackposter/track.py index 590419df79b..840293939c8 100644 --- a/run_page/gpxtrackposter/track.py +++ b/run_page/gpxtrackposter/track.py @@ -289,6 +289,10 @@ def _load_fit_data(self, fit: dict): lng = record["position_long"] / SEMICIRCLE _polylines.append(s2.LatLng.from_degrees(lat, lng)) self.polyline_container.append([lat, lng]) + for record in fit["device_info_mesgs"]: + if "device_index" in record and record["device_index"] == "creator": + self.source = f'{record["manufacturer"]} {record["garmin_product"]} fit' + break if self.polyline_container: self.start_time_local, self.end_time_local = parse_datetime_to_local( self.start_time, self.end_time, self.polyline_container[0] diff --git a/run_page/synced_data_file_logger.py b/run_page/synced_data_file_logger.py index 5a7fb98efce..1b0c6edbe45 100644 --- a/run_page/synced_data_file_logger.py +++ b/run_page/synced_data_file_logger.py @@ -1,5 +1,5 @@ import os -from config import SYNCED_FILE, SYNCED_ACTIVITY_FILE +from config import SYNCED_FILE, SYNCED_ACTIVITY_FILE, NAME_MAPPING_FILE import json @@ -39,3 +39,20 @@ def load_synced_activity_list(): pass return [] + + +def load_fit_name_mapping(): + if os.path.exists(NAME_MAPPING_FILE): + with open(NAME_MAPPING_FILE, "r") as f: + try: + return json.load(f) + except Exception as e: + print(f"json load {NAME_MAPPING_FILE} \nerror {e}") + pass + + return {} + + +def save_fit_name_mapping(name_mapping: dict): + with open(NAME_MAPPING_FILE, "w") as f: + json.dump(name_mapping, f)