diff --git a/run_page/garmin_sync.py b/run_page/garmin_sync.py index 1defb34a4fa..42b725133ce 100755 --- a/run_page/garmin_sync.py +++ b/run_page/garmin_sync.py @@ -108,6 +108,13 @@ async def get_activities(self, start, limit): url = url + "&activityType=running" return await self.fetch_data(url) + async def get_activity_summary(self, activity_id): + """ + Fetch activity summary + """ + url = f"{self.modern_url}/activity-service/activity/{activity_id}" + return await self.fetch_data(url) + async def download_activity(self, activity_id, file_type="gpx"): url = f"{self.modern_url}/download-service/export/{file_type}/activity/{activity_id}" if file_type == "fit": @@ -287,6 +294,16 @@ async def download_new_activities( to_generate_garmin_ids = list(set(activity_ids) - set(downloaded_ids)) print(f"{len(to_generate_garmin_ids)} new activities to be downloaded") + to_generate_garmin_id2title = {} + for id in to_generate_garmin_ids: + try: + activity_summary = await client.get_activity_summary(id) + activity_title = activity_summary.get("activityName", "") + to_generate_garmin_id2title[id] = activity_title + except Exception as e: + print(f"Failed to get activity summary {id}: {str(e)}") + continue + start_time = time.time() await gather_with_concurrency( 10, @@ -298,7 +315,7 @@ async def download_new_activities( print(f"Download finished. Elapsed {time.time()-start_time} seconds") await client.req.aclose() - return to_generate_garmin_ids + return to_generate_garmin_ids, to_generate_garmin_id2title if __name__ == "__main__": @@ -350,6 +367,14 @@ async def download_new_activities( os.mkdir(folder) downloaded_ids = get_downloaded_ids(folder) + if file_type == "fit": + gpx_folder = FOLDER_DICT["gpx"] + if not os.path.exists(gpx_folder): + os.mkdir(gpx_folder) + downloaded_gpx_ids = get_downloaded_ids(gpx_folder) + # merge downloaded_ids:list + downloaded_ids = list(set(downloaded_ids + downloaded_gpx_ids)) + loop = asyncio.get_event_loop() future = asyncio.ensure_future( download_new_activities( @@ -362,7 +387,16 @@ async def download_new_activities( ) ) loop.run_until_complete(future) + new_ids, id2title = future.result() # fit may contain gpx(maybe upload by user) if file_type == "fit": - make_activities_file(SQL_FILE, FOLDER_DICT["gpx"], JSON_FILE, file_suffix="gpx") - make_activities_file(SQL_FILE, folder, JSON_FILE, file_suffix=file_type) + make_activities_file( + SQL_FILE, + FOLDER_DICT["gpx"], + JSON_FILE, + file_suffix="gpx", + activity_title_dict=id2title, + ) + make_activities_file( + SQL_FILE, folder, JSON_FILE, file_suffix=file_type, activity_title_dict=id2title + ) diff --git a/run_page/garmin_sync_cn_global.py b/run_page/garmin_sync_cn_global.py index 98a3ebe42d3..00ccbaaa1ae 100644 --- a/run_page/garmin_sync_cn_global.py +++ b/run_page/garmin_sync_cn_global.py @@ -64,7 +64,7 @@ ) ) loop.run_until_complete(future) - new_ids = future.result() + new_ids, id2title = future.result() to_upload_files = [] for i in new_ids: @@ -89,5 +89,9 @@ # Step 2: # Generate track from fit/gpx file - make_activities_file(SQL_FILE, GPX_FOLDER, JSON_FILE, file_suffix="gpx") - make_activities_file(SQL_FILE, FIT_FOLDER, JSON_FILE, file_suffix="fit") + make_activities_file( + SQL_FILE, GPX_FOLDER, JSON_FILE, file_suffix="gpx", activity_title_dict=id2title + ) + make_activities_file( + SQL_FILE, FIT_FOLDER, JSON_FILE, file_suffix="fit", activity_title_dict=id2title + ) diff --git a/run_page/garmin_to_strava_sync.py b/run_page/garmin_to_strava_sync.py index d7fe8a0cf70..483cd6501b3 100644 --- a/run_page/garmin_to_strava_sync.py +++ b/run_page/garmin_to_strava_sync.py @@ -65,7 +65,7 @@ ) ) loop.run_until_complete(future) - new_ids = future.result() + new_ids, id2title = future.result() print(f"To upload to strava {len(new_ids)} files") index = 1 for i in new_ids: diff --git a/run_page/generator/__init__.py b/run_page/generator/__init__.py index 3a504e17a6e..d1d36a488e4 100644 --- a/run_page/generator/__init__.py +++ b/run_page/generator/__init__.py @@ -76,9 +76,11 @@ def sync(self, force): sys.stdout.flush() self.session.commit() - def sync_from_data_dir(self, data_dir, file_suffix="gpx"): + def sync_from_data_dir(self, data_dir, file_suffix="gpx", activity_title_dict={}): loader = track_loader.TrackLoader() - tracks = loader.load_tracks(data_dir, file_suffix=file_suffix) + tracks = loader.load_tracks( + data_dir, file_suffix=file_suffix, activity_title_dict=activity_title_dict + ) print(f"load {len(tracks)} tracks") if not tracks: print("No tracks found.") @@ -120,6 +122,7 @@ def sync_from_app(self, app_tracks): self.session.commit() def load(self): + # if sub_type is not in the db, just add an empty string to it activities = ( self.session.query(Activity) .filter(Activity.distance > 0.1) diff --git a/run_page/generator/db.py b/run_page/generator/db.py index a153e38c343..c8c83b1f7bf 100644 --- a/run_page/generator/db.py +++ b/run_page/generator/db.py @@ -5,7 +5,16 @@ import geopy from geopy.geocoders import Nominatim -from sqlalchemy import Column, Float, Integer, Interval, String, create_engine +from sqlalchemy import ( + Column, + Float, + Integer, + Interval, + String, + create_engine, + inspect, + text, +) from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker @@ -29,6 +38,7 @@ def randomword(): "distance", "moving_time", "type", + "subtype", "start_date", "start_date_local", "location_country", @@ -47,6 +57,7 @@ class Activity(Base): moving_time = Column(Interval) elapsed_time = Column(Interval) type = Column(String) + subtype = Column(String) start_date = Column(String) start_date_local = Column(String) location_country = Column(String) @@ -106,6 +117,7 @@ def update_or_create_activity(session, run_activity): moving_time=run_activity.moving_time, elapsed_time=run_activity.elapsed_time, type=run_activity.type, + subtype=run_activity.subtype, start_date=run_activity.start_date, start_date_local=run_activity.start_date_local, location_country=location_country, @@ -123,6 +135,7 @@ def update_or_create_activity(session, run_activity): activity.moving_time = run_activity.moving_time activity.elapsed_time = run_activity.elapsed_time activity.type = run_activity.type + activity.subtype = run_activity.subtype activity.average_heartrate = run_activity.average_heartrate activity.average_speed = float(run_activity.average_speed) activity.summary_polyline = ( @@ -135,10 +148,37 @@ def update_or_create_activity(session, run_activity): return created +def add_missing_columns(engine, model): + inspector = inspect(engine) + table_name = model.__tablename__ + columns = {col["name"] for col in inspector.get_columns(table_name)} + missing_columns = [] + + for column in model.__table__.columns: + if column.name not in columns: + missing_columns.append(column) + if missing_columns: + with engine.connect() as conn: + for column in missing_columns: + column_type = str(column.type) + conn.execute( + text( + f"ALTER TABLE {table_name} ADD COLUMN {column.name} {column_type}" + ) + ) + + def init_db(db_path): engine = create_engine( f"sqlite:///{db_path}", connect_args={"check_same_thread": False} ) Base.metadata.create_all(engine) - session = sessionmaker(bind=engine) - return session() + + # check missing columns + add_missing_columns(engine, Activity) + + sm = sessionmaker(bind=engine) + session = sm() + # apply the changes + session.commit() + return session diff --git a/run_page/gpxtrackposter/track.py b/run_page/gpxtrackposter/track.py index 90dfda33585..8e07516b53a 100644 --- a/run_page/gpxtrackposter/track.py +++ b/run_page/gpxtrackposter/track.py @@ -41,6 +41,7 @@ def __init__(self): self.file_names = [] self.polylines = [] self.polyline_str = "" + self.track_name = None self.start_time = None self.end_time = None self.start_time_local = None @@ -52,6 +53,7 @@ def __init__(self): self.run_id = 0 self.start_latlng = [] self.type = "Run" + self.subtype = None # for fit file self.device = "" def load_gpx(self, file_name): @@ -190,6 +192,8 @@ def _load_gpx_data(self, gpx): polyline_container = [] heart_rate_list = [] for t in gpx.tracks: + if self.track_name is None: + self.track_name = t.name for s in t.segments: try: extensions = [ @@ -246,7 +250,11 @@ def _load_fit_data(self, fit: dict): self.average_heartrate = ( message["avg_heart_rate"] if "avg_heart_rate" in message else None ) - self.type = message["sport"].lower() + if message["sport"].lower() == "running": + self.type = "Run" + else: + self.type = message["sport"].lower() + self.subtype = message["sub_sport"] if "sub_sport" in message else None # moving_dict self.moving_dict["distance"] = message["total_distance"] @@ -333,12 +341,9 @@ def _get_moving_data(gpx): def to_namedtuple(self, run_from="gpx"): d = { "id": self.run_id, - "name": ( - f"run from {run_from} by {self.device}" - if self.device - else f"run from {run_from}" - ), # maybe change later - "type": "Run", # Run for now only support run for now maybe change later + "name": (self.track_name if self.track_name else ""), # maybe change later + "type": self.type, + "subtype": (self.subtype if self.subtype else ""), "start_date": self.start_time.strftime("%Y-%m-%d %H:%M:%S"), "end": self.end_time.strftime("%Y-%m-%d %H:%M:%S"), "start_date_local": self.start_time_local.strftime("%Y-%m-%d %H:%M:%S"), diff --git a/run_page/gpxtrackposter/track_loader.py b/run_page/gpxtrackposter/track_loader.py index 260b92af03b..b3cf26bdbe6 100644 --- a/run_page/gpxtrackposter/track_loader.py +++ b/run_page/gpxtrackposter/track_loader.py @@ -24,24 +24,33 @@ log = logging.getLogger(__name__) -def load_gpx_file(file_name): +def load_gpx_file(file_name, activity_title_dict={}): """Load an individual GPX file as a track by using Track.load_gpx()""" t = Track() t.load_gpx(file_name) + file_id = os.path.basename(file_name).split(".")[0] + if activity_title_dict: + t.track_name = activity_title_dict.get(file_id, t.track_name) return t -def load_tcx_file(file_name): +def load_tcx_file(file_name, activity_title_dict={}): """Load an individual TCX file as a track by using Track.load_tcx()""" t = Track() t.load_tcx(file_name) + file_id = os.path.basename(file_name).split(".")[0] + if activity_title_dict: + t.track_name = activity_title_dict.get(file_id, t.track_name) return t -def load_fit_file(file_name): +def load_fit_file(file_name, activity_title_dict={}): """Load an individual FIT file as a track by using Track.load_fit()""" t = Track() t.load_fit(file_name) + file_id = os.path.basename(file_name).split(".")[0] + if activity_title_dict: + t.track_name = activity_title_dict.get(file_id, t.track_name) return t @@ -66,7 +75,7 @@ def __init__(self): "fit": load_fit_file, } - def load_tracks(self, data_dir, file_suffix="gpx"): + def load_tracks(self, data_dir, file_suffix="gpx", activity_title_dict={}): """Load tracks data_dir and return as a List of tracks""" file_names = [x for x in self._list_data_files(data_dir, file_suffix)] print(f"{file_suffix.upper()} files: {len(file_names)}") @@ -74,7 +83,9 @@ def load_tracks(self, data_dir, file_suffix="gpx"): tracks = [] loaded_tracks = self._load_data_tracks( - file_names, self.load_func_dict.get(file_suffix, load_gpx_file) + file_names, + self.load_func_dict.get(file_suffix, load_gpx_file), + activity_title_dict, ) tracks.extend(loaded_tracks.values()) @@ -146,14 +157,14 @@ def _merge_tracks(tracks): return merged_tracks @staticmethod - def _load_data_tracks(file_names, load_func=load_gpx_file): + def _load_data_tracks(file_names, load_func=load_gpx_file, activity_title_dict={}): """ TODO refactor with _load_tcx_tracks """ tracks = {} with concurrent.futures.ProcessPoolExecutor() as executor: future_to_file_name = { - executor.submit(load_func, file_name): file_name + executor.submit(load_func, file_name, activity_title_dict): file_name for file_name in file_names } for future in concurrent.futures.as_completed(future_to_file_name): diff --git a/run_page/keep_to_strava_sync.py b/run_page/keep_to_strava_sync.py index ecbd29a31a6..0ddf1ef3787 100644 --- a/run_page/keep_to_strava_sync.py +++ b/run_page/keep_to_strava_sync.py @@ -23,7 +23,6 @@ def run_keep_sync(email, password, keep_sports_data_api, with_download_gpx=False): - if not os.path.exists(KEEP2STRAVA_BK_PATH): file = open(KEEP2STRAVA_BK_PATH, "w") file.close() diff --git a/run_page/utils.py b/run_page/utils.py index de6945ee48a..2bfd1780d86 100644 --- a/run_page/utils.py +++ b/run_page/utils.py @@ -48,9 +48,13 @@ def to_date(ts): raise ValueError(f"cannot parse timestamp {ts} into date with fmts: {ts_fmts}") -def make_activities_file(sql_file, data_dir, json_file, file_suffix="gpx"): +def make_activities_file( + sql_file, data_dir, json_file, file_suffix="gpx", activity_title_dict={} +): generator = Generator(sql_file) - generator.sync_from_data_dir(data_dir, file_suffix=file_suffix) + generator.sync_from_data_dir( + data_dir, file_suffix=file_suffix, activity_title_dict=activity_title_dict + ) activities_list = generator.load() with open(json_file, "w") as f: json.dump(activities_list, f) diff --git a/src/utils/const.ts b/src/utils/const.ts index 1836b1be55f..2296406176f 100644 --- a/src/utils/const.ts +++ b/src/utils/const.ts @@ -44,6 +44,8 @@ const PRIVACY_MODE = false; // update for now 2024/11/17 the lights on default is false //set to `false` if you want to make light off as default, only effect when `PRIVACY_MODE` = false const LIGHTS_ON =false; +// richer title for the activity types (like garmin style) +const RICH_TITLE = false; // IF you outside China please make sure IS_CHINESE = false const IS_CHINESE = true; @@ -68,6 +70,23 @@ const MIDDAY_RUN_TITLE = IS_CHINESE ? '午间跑步' : 'Midday Run'; const AFTERNOON_RUN_TITLE = IS_CHINESE ? '午后跑步' : 'Afternoon Run'; const EVENING_RUN_TITLE = IS_CHINESE ? '傍晚跑步' : 'Evening Run'; const NIGHT_RUN_TITLE = IS_CHINESE ? '夜晚跑步' : 'Night Run'; +const RUN_GENERIC_TITLE = IS_CHINESE ? '跑步' : 'Run'; +const RUN_TRAIL_TITLE = IS_CHINESE ? '越野跑' : 'Trail Run'; +const RUN_TREADMILL_TITLE = IS_CHINESE ? '跑步机' : 'Treadmill Run'; +const HIKING_TITLE = IS_CHINESE ? '徒步' : 'Hiking'; +const CYCLING_TITLE = IS_CHINESE ? '骑行' : 'Cycling'; +const SKIING_TITLE = IS_CHINESE ? '滑雪' : 'Skiing'; +const WALKING_TITLE = IS_CHINESE ? '步行' : 'Walking'; + +const ACTIVITY_TYPES = { + RUN_GENERIC_TITLE, + RUN_TRAIL_TITLE, + RUN_TREADMILL_TITLE, + HIKING_TITLE, + CYCLING_TITLE, + SKIING_TITLE, + WALKING_TITLE, +} const RUN_TITLES = { FULL_MARATHON_RUN_TITLE, @@ -97,6 +116,8 @@ export { MAP_HEIGHT, PRIVACY_MODE, LIGHTS_ON, + RICH_TITLE, + ACTIVITY_TYPES, }; const nike = 'rgb(224,237,94)'; // if you want change the main color change here src/styles/variables.scss diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 9470b7aa827..ea3818b480d 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -4,7 +4,7 @@ import { WebMercatorViewport } from 'viewport-mercator-project'; import { chinaGeojson, RPGeometry } from '@/static/run_countries'; import worldGeoJson from '@surbowl/world-geo-json-zh/world.zh.json'; import { chinaCities } from '@/static/city'; -import { MAIN_COLOR, MUNICIPALITY_CITIES_ARR, NEED_FIX_MAP, RUN_TITLES } from './const'; +import { MAIN_COLOR, MUNICIPALITY_CITIES_ARR, NEED_FIX_MAP, RUN_TITLES, ACTIVITY_TYPES, RICH_TITLE } from './const'; import { FeatureCollection, LineString } from 'geojson'; export type Coordinate = [number, number]; @@ -17,6 +17,7 @@ export interface Activity { distance: number; moving_time: string; type: string; + subtype: string; start_date: string; start_date_local: string; location_country?: string | null; @@ -80,11 +81,10 @@ const scrollToMap = () => { } }; -const pattern = /([\u4e00-\u9fa5]{2,}(市|自治州|特别行政区))/g; -const extractLocations = (str: string): string[] => { +const extractCities = (str: string): string[] => { const locations = []; let match; - + const pattern = /([\u4e00-\u9fa5]{2,}(市|自治州|特别行政区|盟|地区))/g; while ((match = pattern.exec(str)) !== null) { locations.push(match[0]); } @@ -92,6 +92,17 @@ const extractLocations = (str: string): string[] => { return locations; }; +const extractDistricts = (str: string): string[] => { + const locations = []; + let match; + const pattern = /([\u4e00-\u9fa5]{2,}(区|县))/g; + while ((match = pattern.exec(str)) !== null) { + locations.push(match[0]); + } + + return locations; +} + const extractCoordinate = (str: string): [number, number] | null => { const pattern = /'latitude': ([-]?\d+\.\d+).*?'longitude': ([-]?\d+\.\d+)/; const match = str.match(pattern); @@ -125,7 +136,7 @@ const locationForRun = ( if (location) { // Only for Chinese now // should filter 臺灣 - const cityMatch = extractLocations(location); + const cityMatch = extractCities(location); const provinceMatch = location.match(/[\u4e00-\u9fa5]{2,}(省|自治区)/); if (cityMatch) { @@ -154,6 +165,12 @@ const locationForRun = ( } if (MUNICIPALITY_CITIES_ARR.includes(city)) { province = city; + if (location) { + const districtMatch = extractDistricts(location); + if (districtMatch.length > 0) { + city = districtMatch[districtMatch.length - 1]; + } + } } const r = { country, province, city, coordinate }; @@ -216,7 +233,51 @@ const geoJsonForMap = (): FeatureCollection => ({ features: worldGeoJson.features.concat(chinaGeojson.features), }) +const getActivitySport = (act: Activity): string => { + if (act.type === 'Run') { + if (act.subtype === 'generic') { + const runDistance = act.distance / 1000; + if (runDistance > 20 && runDistance < 40) { + return RUN_TITLES.HALF_MARATHON_RUN_TITLE; + } else if (runDistance >= 40) { + return RUN_TITLES.FULL_MARATHON_RUN_TITLE; + } + return ACTIVITY_TYPES.RUN_GENERIC_TITLE; + } + else if (act.subtype === 'trail') return ACTIVITY_TYPES.RUN_TRAIL_TITLE; + else if (act.subtype === 'treadmill') return ACTIVITY_TYPES.RUN_TREADMILL_TITLE; + else return ACTIVITY_TYPES.RUN_GENERIC_TITLE; + } + else if (act.type === 'hiking') { + return ACTIVITY_TYPES.HIKING_TITLE; + } + else if (act.type === 'cycling') { + return ACTIVITY_TYPES.CYCLING_TITLE; + } + else if (act.type === 'walking') { + return ACTIVITY_TYPES.WALKING_TITLE; + } + // if act.type contains 'skiing' + else if (act.type.includes('skiing')) { + return ACTIVITY_TYPES.SKIING_TITLE; + } + return ""; +} + const titleForRun = (run: Activity): string => { + if (RICH_TITLE) { + // 1. try to use user defined name + if (run.name != '') { + return run.name; + } + // 2. try to use location+type if the location is available, eg. 'Shanghai Run' + const { city, province } = locationForRun(run); + const activity_sport = getActivitySport(run); + if (city && city.length > 0 && activity_sport.length > 0) { + return `${city} ${activity_sport}`; + } + } + // 3. use time+length if location or type is not available const runDistance = run.distance / 1000; const runHour = +run.start_date_local.slice(11, 13); if (runDistance > 20 && runDistance < 40) {