Merge pull request #147 from doZennn/fix-steam-ci

CI: Steam appid checks via Steam instead of web apis

.github/scripts/check_gamefixes.py

@@ -5,10 +5,12 @@
 from urllib.request import urlopen, Request
 from http.client import HTTPSConnection
 from typing import Any
-from collections.abc import Iterator, Generator
+from collections.abc import Generator
 
 import ijson
 
+from steam_client import Steam
+
 # Represents a valid API endpoint, where the first element is the host, second
 # is the url (e.g., store.steampowered.com and store.steampowered.com). The API
 # endpoint will be used to validate local gamefix modules IDs against. Assumes
@@ -22,64 +24,27 @@
     'Accept-Language': 'en-US,en;q=0.5',
 }
 
-# Steam games that are no longer on sale or only in certain regions, but are valid IDs
-whitelist_steam = {231990, 4730, 105400, 321040, 12840, 7850, 1175730}
-
-
-def check_steamfixes(project: Path, url: str, api: ApiEndpoint) -> None:
+def check_steamfixes(project: Path) -> None:
     """Verifies if the name of Steam gamefix modules are valid entries.
 
     Raises a ValueError if the ID is not found upstream
     """
-    appids = set()
-
-    # Get all IDs
-    for file in project.joinpath('gamefixes-steam').glob('*'):
-        appid = file.name.removesuffix('.py')
-        if not appid.isnumeric():
-            continue
-        appids.add(int(appid))
-
-    # Check the IDs against ours
-    print(f"Validating Steam app ids against '{url}'...", file=sys.stderr)
-    with urlopen(Request(url, headers=headers), timeout=500) as r:
-        for obj in ijson.items(r, 'applist.apps.item'):
-            if obj['appid'] in appids:
-                print(f'Removing Steam app id: "{obj["appid"]}"', file=sys.stderr)
-                appids.remove(obj['appid'])
-            if not appids:
-                break
-
-    # Double check that the ID is valid. It's possible that it is but
-    # wasn't returned from the api in `url` for some reason
-    if appids:
-        host, endpoint = api
-        conn = HTTPSConnection(host)
-
-        print(f"Validating Steam app ids against '{host}'...", file=sys.stderr)
-        for appid in appids.copy():
-            conn.request('GET', f'{endpoint}{appid}')
-            r = conn.getresponse()
-            parser: Iterator[tuple[str, str, Any]] = ijson.parse(r)
+    steam = Steam()
 
-            for prefix, _, value in parser:
-                if prefix == f'{appid}.success' and isinstance(value, bool) and value:
-                    print(f'Removing Steam app id: "{appid}"', file=sys.stderr)
-                    appids.remove(appid)
-                    break
+    invalid_appids = set()
 
-            if not appids:
-                break
+    for appids in _batch_generator(project.joinpath('gamefixes-steam'), 50):
+        appids = {int(x) for x in appids}
 
-            r.read()
+        steam_appids = steam.get_valid_appids(appids)
+
+        for appid in steam_appids:
+            if appid not in appids:
+                invalid_appids.add(appid)
 
-        conn.close()
-
-    print(f'Remaining Steam app ids: {appids}', file=sys.stderr)
-    for appid in appids:
-        if appid not in whitelist_steam:
-            err = f'Steam app id is invalid: {appid}'
-            raise ValueError(err)
+    if invalid_appids:
+        err = f'The following Steam app ids are invalid: {invalid_appids}'
+        raise ValueError(err)
 
 
 def check_gogfixes(project: Path, url: str, api: ApiEndpoint) -> None:
@@ -145,15 +110,20 @@ def check_gogfixes(project: Path, url: str, api: ApiEndpoint) -> None:
 
 
 def _batch_generator(gamefix: Path, size: int = 50) -> Generator[set[str], Any, Any]:
+    is_steam = 'gamefixes-steam' in gamefix.name
     appids = set()
     # Keep track of the count because some APIs enforce limits
     count = 0
 
     # Process only umu-* app ids
     for file in gamefix.glob('*'):
-        if not file.name.startswith('umu-'):
+        if not file.name.startswith('umu-') and not is_steam:
             continue
+
         appid = file.name.removeprefix('umu-').removesuffix('.py')
+        if is_steam and not appid.isnumeric():
+            continue
+
         appids.add(appid)
         if count == size:
             yield appids
@@ -214,33 +184,17 @@ def main() -> None:
     project = Path(__file__).parent.parent.parent
     print(project)
 
-    # Steam API to acquire a single id. Used as fallback in case some IDs could
-    # not be validated. Unforutnately, this endpoint does not accept a comma
-    # separated list of IDs so we have to make one request per ID after making
-    # making a request to `api.steampowered.com`.
-    # NOTE: There's neither official nor unofficial documentation. Only forum posts
-    # See https://stackoverflow.com/questions/46330864/steam-api-all-games
-    steamapi: ApiEndpoint = ('store.steampowered.com', '/api/appdetails?appids=')
-
     # UMU Database, that will be used to validate umu gamefixes ids against
     # See https://github.com/Open-Wine-Components/umu-database/blob/main/README.md
     umudb_gog: ApiEndpoint = ('umu.openwinecomponents.org', '/umu_api.php?store=gog')
 
-    # Steam API
-    # Main API used to validate steam gamefixes
-    # NOTE: There's neither official nor unofficial documentation. Only forum posts
-    # See https://stackoverflow.com/questions/46330864/steam-api-all-games
-    steampowered = (
-        'https://api.steampowered.com/ISteamApps/GetAppList/v0002/?format=json'
-    )
-
     # GOG API
     # See https://gogapidocs.readthedocs.io/en/latest/galaxy.html#get--products
     gogapi = 'https://api.gog.com/products?ids='
 
     check_links(project)
     check_filenames(project)
-    check_steamfixes(project, steampowered, steamapi)
+    check_steamfixes(project)
     check_gogfixes(project, gogapi, umudb_gog)
 
 

.github/scripts/steam_client.py

@@ -0,0 +1,68 @@
+"""Steam Client"""
+
+import sys
+from steam.client import SteamClient
+from steam.core.msg import MsgProto
+from steam.enums import EResult
+from steam.enums.emsg import EMsg
+from steam.utils.proto import proto_to_dict
+from steam.core.connection import WebsocketConnection
+
+class Steam: # noqa: D101
+    def __init__(self) -> None: # noqa: D107
+        self.logged_on_once = False
+
+        self.steam = client = SteamClient()
+        client.connection = WebsocketConnection()
+
+        @client.on('error')
+        def handle_error(result: EResult) -> None:
+            raise ValueError(f'Steam error: {repr(result)}')
+
+        @client.on('connected')
+        def handle_connected() -> None:
+            print(f'Connected to {client.current_server_addr}', file=sys.stderr)
+
+        @client.on('channel_secured')
+        def send_login() -> None:
+            if self.logged_on_once and self.steam.relogin_available:
+                self.steam.relogin()
+
+        @client.on('disconnected')
+        def handle_disconnect() -> None:
+            print('Steam disconnected', file=sys.stderr)
+            if self.logged_on_once:
+                print('Reconnecting...', file=sys.stderr)
+                client.reconnect(maxdelay=30)
+
+        @client.on('logged_on')
+        def handle_after_logon() -> None:
+            self.logged_on_once = True
+
+        client.anonymous_login()
+
+    def get_valid_appids(self, appids: set[int]) -> list[int]:
+        """Queries Steam for the specified appids.
+
+        If an appid doesn't exist, it won't be in the response.
+
+        Raises a ValueError if Steam returns unexpected data
+        """
+        # https://github.com/SteamRE/SteamKit/blob/master/SteamKit2/SteamKit2/Base/Generated/SteamMsgClientServerAppInfo.cs#L331
+        resp = self.steam.send_job_and_wait(
+            message = MsgProto(EMsg.ClientPICSProductInfoRequest),
+            body_params = {
+                'apps': map(lambda x: {'appid': x}, appids),
+                'meta_data_only': True
+            },
+            timeout=15
+        )
+
+        if not resp:
+            err = 'Error retrieving appinfo from Steam'
+            raise ValueError(err)
+
+        data = proto_to_dict(resp)
+        appids = [app['appid'] for app in data['apps']]
+
+        return appids

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
           python3 -m pip install --upgrade pip
           pip install ruff
           pip install ijson
+          pip install "git+https://github.com/njbooher/steam.git@wsproto#egg=steam[client]"
       - name: Lint with Shellcheck
         run: |
           shellcheck winetricks