New rest endpoint for tickers

Dockerfile

@@ -1,4 +1,4 @@
-FROM    python:3.8
+FROM    python:3.11
 RUN     useradd -ms /bin/bash app
 USER    app
 WORKDIR /home/app

Jenkinsfile

@@ -3,7 +3,7 @@
 def pipeline = new org.docker.AppPipeline(steps: this,
     dockerImageName: 'sora2/pricing-server',
     dockerRegistryCred: 'bot-sora2-rw',
-    secretScannerExclusion: '.*docker-compose.yml',
+    secretScannerExclusion: '.*docker-compose.yml\$|.*docker-compose-external.yml\$',
     dockerImageTags: ['master':'latest'],
     deepSecretScannerExclusion: ["web.py", "alembic.ini"],
     gitUpdateSubmodule: true)

alembic/versions/a93bf2ae653f_added_liquidity_to_pairs.py

@@ -0,0 +1,30 @@
+"""Added liquidity to pairs
+
+Revision ID: a93bf2ae653f
+Revises: 5b71ff3399c8
+Create Date: 2024-12-24 17:24:19.293241
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a93bf2ae653f'
+down_revision = '5b71ff3399c8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('pair', sa.Column('from_token_liquidity', sa.Numeric(), nullable=True))
+    op.add_column('pair', sa.Column('to_token_liquidity', sa.Numeric(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('pair', 'to_token_liquidity')
+    op.drop_column('pair', 'from_token_liquidity')
+    # ### end Alembic commands ###

docker-compose-external.yml

@@ -0,0 +1,27 @@
+version: "3.4"
+volumes:
+  chain:
+
+services:
+  postgres:
+    image: postgres
+    environment:
+      POSTGRES_DB: "${DB_NAME}"
+      POSTGRES_USER: "${DB_USER}"
+      POSTGRES_PASSWORD: "${DB_PASSWORD}"
+    ports:
+      - "127.0.0.1:5432:5432"
+  import:
+    build: .
+    depends_on:
+      - "postgres"
+    command: 
+      sh -c "sleep 1 &&
+      env SUBSTRATE_URL=${SUBSTRATE_URL} DATABASE_URL=postgresql+asyncpg://${DB_USER}:${DB_PASSWORD}@postgres/${DB_NAME} sh start.sh"
+  web:
+    build: .
+    depends_on:
+      - "postgres"
+    ports:
+      - "8000:8000"
+    command: "env DATABASE_URL=postgresql+asyncpg://${DB_USER}:${DB_PASSWORD}@postgres/${DB_NAME} gunicorn -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0 web:app"

models.py

@@ -38,6 +38,8 @@ class Pair(Base):
         foreign_keys=[to_token_id],
         backref=backref("to_pairs", uselist=True, cascade="delete,all"),
     )
+    from_token_liquidity = Column(Numeric())
+    to_token_liquidity = Column(Numeric())
     quote_price = Column(Numeric(), nullable=True)
 
 

processing.py

@@ -22,6 +22,7 @@
 ETH_ID = "0x0200070000000000000000000000000000000000000000000000000000000000"
 KUSD_ID = "0x02000c0000000000000000000000000000000000000000000000000000000000"
 VXOR_ID = "0x006a271832f44c93bd8692584d85415f0f3dccef9748fecd129442c8edcb4361"
+DAI_ID = "0x0200060000000000000000000000000000000000000000000000000000000000"
 TECH_ACCOUNT = (
     # "0x54734f90f971a02c609b2d684e61b5574e35ac9942579a2635aada58e5d836a7"  # noqa
     "cnTQ1kbv7PBNNQrEb1tZpmK7ftiv4yCCpUQy1J2y7Y54Taiaw"  # noqa

requirements.in

@@ -7,7 +7,9 @@ graphene-sqlalchemy
 gunicorn
 python-decouple
 scalecodec
-sqlalchemy
+sqlalchemy==1.4.37
 substrate-interface
 tqdm
 uvicorn
+starlette==0.14.2
+greenlet

requirements.txt

@@ -1,31 +1,47 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
-#    pip-compile
+#    pip-compile requirements.in
 #
-aiosqlite==0.17.0
+aiosqlite==0.20.0
     # via -r requirements.in
-alembic==1.7.4
+alembic==1.14.1
     # via -r requirements.in
 aniso8601==7.0.0
     # via graphene
-asgiref==3.4.1
-    # via uvicorn
-asyncpg==0.24.0
+asyncpg==0.30.0
     # via -r requirements.in
-base58==2.1.0
+base58==2.1.1
     # via
     #   scalecodec
     #   substrate-interface
-certifi==2021.10.8
+certifi==2024.12.14
     # via
     #   requests
     #   substrate-interface
-charset-normalizer==2.0.7
+cffi==1.17.1
+    # via pynacl
+charset-normalizer==3.4.1
     # via requests
-click==8.0.3
+click==8.1.8
     # via uvicorn
+cytoolz==1.0.1
+    # via eth-utils
+ecdsa==0.19.0
+    # via substrate-interface
+eth-hash==0.7.1
+    # via eth-utils
+eth-keys==0.6.1
+    # via substrate-interface
+eth-typing==5.1.0
+    # via
+    #   eth-keys
+    #   eth-utils
+eth-utils==5.2.0
+    # via
+    #   eth-keys
+    #   substrate-interface
 fastapi==0.68.2
     # via -r requirements.in
 graphene==2.1.9
@@ -40,84 +56,93 @@ graphql-core==2.3.2
     #   graphql-relay
 graphql-relay==2.0.1
     # via graphene
-greenlet==1.1.2
-    # via sqlalchemy
-gunicorn==20.1.0
+greenlet==3.1.1
+    # via -r requirements.in
+gunicorn==23.0.0
     # via -r requirements.in
-h11==0.12.0
+h11==0.14.0
     # via uvicorn
-idna==3.3
+idna==3.10
     # via
     #   requests
     #   substrate-interface
-mako==1.1.5
+mako==1.3.8
     # via alembic
-markupsafe==2.0.1
+markupsafe==3.0.2
     # via mako
-more-itertools==8.10.0
+more-itertools==10.6.0
     # via scalecodec
+packaging==24.2
+    # via gunicorn
 promise==2.3
     # via
     #   graphene-sqlalchemy
     #   graphql-core
     #   graphql-relay
-py-bip39-bindings==0.1.9
+py-bip39-bindings==0.2.0
     # via substrate-interface
-py-ed25519-bindings==1.0
+py-ed25519-zebra-bindings==1.2.0
     # via substrate-interface
-py-sr25519-bindings==0.2.0
+py-sr25519-bindings==0.2.1
     # via substrate-interface
-pydantic==1.8.2
+pycparser==2.22
+    # via cffi
+pycryptodome==3.21.0
+    # via substrate-interface
+pydantic==1.10.21
     # via fastapi
-python-decouple==3.5
+pynacl==1.5.0
+    # via substrate-interface
+python-decouple==3.8
     # via -r requirements.in
-requests==2.26.0
+requests==2.32.3
     # via
     #   scalecodec
     #   substrate-interface
-rx==1.6.1
+rx==1.6.3
     # via graphql-core
-scalecodec==1.2.4
+scalecodec==1.2.11
     # via
     #   -r requirements.in
     #   substrate-interface
 singledispatch==3.7.0
     # via graphene-sqlalchemy
-six==1.16.0
+six==1.17.0
     # via
+    #   ecdsa
     #   graphene
     #   graphene-sqlalchemy
     #   graphql-core
     #   graphql-relay
     #   promise
+    #   python-dateutil
     #   singledispatch
-    #   websocket-client
-sqlalchemy==1.4.26
+sqlalchemy==1.4.37
     # via
     #   -r requirements.in
     #   alembic
     #   graphene-sqlalchemy
 starlette==0.14.2
-    # via fastapi
-substrate-interface==1.7.1
+    # via
+    #   -r requirements.in
+    #   fastapi
+substrate-interface==1.7.11
     # via -r requirements.in
-tqdm==4.62.3
+toolz==1.0.0
+    # via cytoolz
+tqdm==4.67.1
     # via -r requirements.in
-typing-extensions==3.10.0.2
+typing-extensions==4.12.2
     # via
     #   aiosqlite
+    #   alembic
+    #   eth-typing
     #   pydantic
-urllib3==1.26.7
+urllib3==2.3.0
     # via requests
-uvicorn==0.15.0
+uvicorn==0.34.0
     # via -r requirements.in
-websocket-client==0.59.0
+websocket-client==1.8.0
     # via substrate-interface
-xxhash==2.0.2
+xxhash==3.5.0
     # via substrate-interface
-
-eth-utils==1.3.0
-    # fix substrateinterface import error
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

run_node_processing.py

@@ -37,6 +37,8 @@
 
 SWAP_FEE_ASSETS = {}
 
+POLL_INTERVAL = 30
+
 # BLOCK_IMPORT_LIMIT = 10 # In blocks, 0, None or float("inf") - to not stop
 
 # WAIT_FOR_NEXT_IMPORT = 4 # In seconds
@@ -193,6 +195,44 @@ async def get_all_pairs(session):
         pairs[p.from_token.id, p.to_token.id] = p
     return pairs
 
+async def update_all_pairs_liquidity(session, substrate):
+    try:
+        result = await session.execute(select(Pair))
+
+        pairs = result.scalars().all() 
+
+        for pair in pairs:
+            result = substrate.query(
+                module="PoolXYK",
+                storage_function="Reserves",
+                params=[
+                    "0x" + hex(int(pair.from_token_id))[2:].zfill(64),
+                    "0x" + hex(int(pair.to_token_id))[2:].zfill(64),
+                ],
+            )
+
+            result_rev = substrate.query(
+                module="PoolXYK",
+                storage_function="Reserves",
+                params=[
+                    "0x" + hex(int(pair.to_token_id))[2:].zfill(64),
+                    "0x" + hex(int(pair.from_token_id))[2:].zfill(64),
+                ],
+            )
+
+            if result:
+                liquidity_from, liquidity_to = result.value
+                pair.from_token_liquidity = liquidity_from / DENOM
+                pair.to_token_liquidity = liquidity_to / DENOM
+            if result_rev:
+                liquidity_to, liquidity_from = result_rev.value
+                pair.from_token_liquidity += liquidity_from / DENOM
+                pair.to_token_liquidity += liquidity_to / DENOM
+
+        await session.commit()
+    except Exception as e:
+        await session.rollback()
+        logging.error(f"Error updating liquidity for all pairs: {e}")
 
 async def update_volumes(session):
     """
@@ -584,6 +624,7 @@ def get_end(substrate: SubstrateInterface):
         if not silent:
             logging.info("Updating trade volumes...")
         await update_volumes(session)
+        await update_all_pairs_liquidity(session, substrate)
 
 
 async def async_main_loop(async_session, args):
@@ -594,7 +635,7 @@ async def async_main_loop(async_session, args):
         await async_main(async_session, args.begin, args.clean, args.silent)
         if not args.silent:
             logging.info("Waiting for new blocks...")
-        await asyncio.sleep(decouple.config("POLL_INTERVAL", default=600, cast=int))
+        await asyncio.sleep(decouple.config("POLL_INTERVAL", default=POLL_INTERVAL, cast=int))
 
 
 if __name__ == "__main__":