feat: market-sim monorepo (#696)

* feat: add vega_query * feat: add vega_query tests * feat: add vega_protos * feat: add vega_protos tests * chore: rename vega_python_protos usages * chore: add support for new packages * chore: update dependencies * fix: failing tests
vegaprotocol · Aug 27, 2024 · 4082e63 · 4082e63
1 parent 9e6bee3
commit 4082e63
Show file tree

Hide file tree

Showing 217 changed files with 41,491 additions and 321 deletions.
diff --git a/Makefile b/Makefile
@@ -85,6 +85,22 @@ build_deps_capsule:
 	@mkdir -p ./vega_sim/bin
 	cd ${EXTERN_DIR}/vegacapsule && go build -o ../../vega_sim/bin/ ./...
 
+protos:
+	@if [ ! -d ./extern/ ]; then mkdir ./extern/; fi
+	@echo "Downloading Git dependencies into " ${EXTERN_DIR}
+	@echo "Downloading Vega"
+	@if [ ! -d ./extern/vega ]; then mkdir ./extern/vega; git clone https://github.com/vegaprotocol/vega ${EXTERN_DIR}/vega; fi
+ifneq (${VEGA_TAG},develop)
+	@git -C ${EXTERN_DIR}/vega pull; git -C ${EXTERN_DIR}/vega checkout ${VEGA_TAG}
+else
+	@git -C ${EXTERN_DIR}/vega checkout develop; git -C ${EXTERN_DIR}/vega pull
+endif
+	@rm -rf ./vega_protos/protos
+	@mkdir ./vega_protos/protos
+	@buf generate extern/vega/protos/sources --template ./vega_protos/buf.gen.yaml
+	@GENERATED_DIR=./vega_protos/protos vega_protos/post-generate.sh
+	@black .
+
 .PHONY: black
 black:
 	@black .
@@ -103,12 +119,20 @@ flake8:
 
 .PHONY: test
 test:
-	@env PYTHONPATH=. pytest -m "not integration and not api" tests/
+	@env PYTHONPATH=. pytest -m "not (integration or api or vega_query or vega_protos)" tests/
 
 .PHONY: test_api
 test_api:
 	@env PYTHONPATH=. pytest -m api tests/
 
+.PHONY: test_protos
+test_protos:
+	@env PYTHONPATH=. pytest tests/vega_protos
+
+.PHONY: test_query
+test_query:
+	@env PYTHONPATH=. pytest tests/vega_query
+
 .PHONY: test_integration
 test_integration:
 	@env PYTHONPATH=. pytest -m integration tests/

diff --git a/examples/agent_market/agents.py b/examples/agent_market/agents.py
@@ -6,8 +6,8 @@
 from vega_sim.environment import StateAgent, VegaState
 from vega_sim.environment.agent import StateAgentWithWallet
 from vega_sim.null_service import VegaServiceNull
-from vega_python_protos.protos.vega import markets as markets_protos
-from vega_python_protos.protos.vega import vega as vega_protos, governance as gov_protos
+from vega_protos.protos.vega import markets as markets_protos
+from vega_protos.protos.vega import vega as vega_protos, governance as gov_protos
 from vega_sim.service import VegaService
 
 WalletConfig = namedtuple("WalletConfig", ["name", "passphrase"])

diff --git a/examples/notebooks/InvestigatePriceMove.ipynb b/examples/notebooks/InvestigatePriceMove.ipynb
@@ -11,7 +11,7 @@
     "import numpy as np\n",
     "\n",
     "import vega_sim.api.data_raw as data_raw\n",
-    "import vega_python_protos.protos.vega as vega_protos\n",
+    "import vega_protos.protos.vega as vega_protos\n",
     "from vega_sim.grpc.client import VegaTradingDataClientV2"
    ]
   },

diff --git a/examples/notebooks/feature_tests/0002-STTL-008.ipynb b/examples/notebooks/feature_tests/0002-STTL-008.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos\n"
+    "import vega_protos.protos.vega as vega_protos\n"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-010.ipynb b/examples/notebooks/feature_tests/0004-AMND-010.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos\n"
+    "import vega_protos.protos.vega as vega_protos\n"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-021.ipynb b/examples/notebooks/feature_tests/0004-AMND-021.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos"
+    "import vega_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-022.ipynb b/examples/notebooks/feature_tests/0004-AMND-022.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos"
+    "import vega_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-023.ipynb b/examples/notebooks/feature_tests/0004-AMND-023.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos"
+    "import vega_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-024.ipynb b/examples/notebooks/feature_tests/0004-AMND-024.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_python_protos.protos.vega as vega_protos"
+    "import vega_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/nullchain.py b/examples/nullchain.py
@@ -5,7 +5,7 @@
 
 from vega_sim.null_service import VegaServiceNull
 from vega_sim.service import MarketStateUpdateType
-import vega_python_protos.protos.vega as vega_protos
+import vega_protos.protos.vega as vega_protos
 
 
 WalletConfig = namedtuple("WalletConfig", ["name", "passphrase"])

diff --git a/examples/visualisations/utils.py b/examples/visualisations/utils.py
@@ -18,7 +18,7 @@
 from vega_sim.service import PeggedOrder
 from vega_sim.api.market import MarketConfig
 
-import vega_python_protos.protos.vega as vega_protos
+import vega_protos.protos.vega as vega_protos
 
 
 PartyConfig = namedtuple("WalletConfig", ["wallet_name", "key_name"])

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -33,6 +33,7 @@ python-dotenv = "*"
 deprecated = "*"
 psutil = "*"
 docker = "*"
+pyyaml = "*"
 websocket-client = "*"
 gymnasium = {version = "*", optional = true}
 sb3-contrib = {version = "^2.0.0a1", optional = true}
@@ -42,7 +43,6 @@ pettingzoo = {version = "*", optional = true}
 numba = {version = "^0.57.1", optional = true}
 tianshou = {version = "*", optional = true}
 protofuzz = { git = "https://github.com/trailofbits/protofuzz", branch="master" }
-vega_python_protos = {git = "https://github.com/vegaprotocol/vega-python-protos", branch="develop" }
 
 [tool.poetry.group.dev.dependencies]
 black = "*"
@@ -53,7 +53,6 @@ Jinja2 = "*"
 nbmake = "*"
 matplotlib = "*"
 pytest-xdist = "*"
-vegapy = {git = "https://github.com/cdummett/vegapy/", branch="develop"}
 
 
 [build-system]

diff --git a/pytest.ini b/pytest.ini
@@ -1,6 +1,8 @@
 [pytest]
 markers = 
-    integration: mark a test as requiring a full vega sim infrastructure with running backend\
+    integration: mark a test as requiring a full vega sim infrastructure with running backend
+    vega_query: tests for checking vega_query package
+    vega_protos: tests for checking vega_protos package
     api: mark a test as checking vega-market-sim api coverage
 
 log_file_format=%(asctime)s.%(msecs)03d %(threadName)s %(processName)s (%(filename)s:%(funcName)s:%(lineno)s):%(message)s

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -67,16 +67,13 @@ pyflakes==3.2.0 ; python_version >= "3.10" and python_version < "3.12"
 pygments==2.18.0 ; python_version >= "3.10" and python_version < "3.12"
 pynacl==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6-qt6==6.7.2 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6-sip==13.8.0 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6==6.7.1 ; python_version >= "3.10" and python_version < "3.12"
 pytest-xdist==3.6.1 ; python_version >= "3.10" and python_version < "3.12"
 pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.12"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "3.12"
 python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "3.12"
 pytz==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 pywin32==306 ; python_version >= "3.10" and python_version < "3.12" and sys_platform == "win32"
-pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "3.12"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.12"
 pyzmq==26.0.3 ; python_version >= "3.10" and python_version < "3.12"
 referencing==0.35.1 ; python_version >= "3.10" and python_version < "3.12"
 requests-mock==1.12.1 ; python_version >= "3.10" and python_version < "3.12"
@@ -94,8 +91,6 @@ traitlets==5.14.3 ; python_version >= "3.10" and python_version < "3.12"
 typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.12"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
-vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@19ccb1c97e5f1d7f2676e89fbb7a814d6172f311 ; python_version >= "3.10" and python_version < "3.12"
-vegapy @ git+https://github.com/cdummett/vegapy/@048a05e186aa33afa881298031e8fe58a9289559 ; python_version >= "3.10" and python_version < "3.12"
 wcwidth==0.2.13 ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 wrapt==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
diff --git a/requirements-learning.txt b/requirements-learning.txt
@@ -94,16 +94,13 @@ pyflakes==3.2.0 ; python_version >= "3.10" and python_version < "3.12"
 pygments==2.18.0 ; python_version >= "3.10" and python_version < "3.12"
 pynacl==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6-qt6==6.7.2 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6-sip==13.8.0 ; python_version >= "3.10" and python_version < "3.12"
-pyqt6==6.7.1 ; python_version >= "3.10" and python_version < "3.12"
 pytest-xdist==3.6.1 ; python_version >= "3.10" and python_version < "3.12"
 pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.12"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "3.12"
 python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "3.12"
 pytz==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 pywin32==306 ; python_version >= "3.10" and python_version < "3.12" and sys_platform == "win32"
-pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "3.12"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.12"
 pyzmq==26.0.3 ; python_version >= "3.10" and python_version < "3.12"
 referencing==0.35.1 ; python_version >= "3.10" and python_version < "3.12"
 requests-mock==1.12.1 ; python_version >= "3.10" and python_version < "3.12"
@@ -130,8 +127,6 @@ triton==2.3.1 ; platform_system == "Linux" and platform_machine == "x86_64" and
 typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.12"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
-vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@19ccb1c97e5f1d7f2676e89fbb7a814d6172f311 ; python_version >= "3.10" and python_version < "3.12"
-vegapy @ git+https://github.com/cdummett/vegapy/@048a05e186aa33afa881298031e8fe58a9289559 ; python_version >= "3.10" and python_version < "3.12"
 wcwidth==0.2.13 ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 werkzeug==3.0.3 ; python_version >= "3.10" and python_version < "3.12"

diff --git a/requirements.txt b/requirements.txt
@@ -1,56 +1,42 @@
-black==24.4.2 ; python_version >= "3.10" and python_version < "3.12"
 certifi==2024.6.2 ; python_version >= "3.10" and python_version < "3.12"
 cffi==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
 charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "3.12"
-click==8.1.7 ; python_version >= "3.10" and python_version < "3.12"
-colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.12" and (sys_platform == "win32" or platform_system == "Windows")
 contourpy==1.2.1 ; python_version >= "3.10" and python_version < "3.12"
 cycler==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
 deprecated==1.2.14 ; python_version >= "3.10" and python_version < "3.12"
 docker==7.1.0 ; python_version >= "3.10" and python_version < "3.12"
-exceptiongroup==1.2.1 ; python_version >= "3.10" and python_version < "3.11"
 fonttools==4.53.0 ; python_version >= "3.10" and python_version < "3.12"
 googleapis-common-protos==1.63.1 ; python_version >= "3.10" and python_version < "3.12"
 grpcio-tools==1.62.2 ; python_version >= "3.10" and python_version < "3.12"
 grpcio==1.64.1 ; python_version >= "3.10" and python_version < "3.12"
 idna==3.7 ; python_version >= "3.10" and python_version < "3.12"
 inflection==0.5.1 ; python_version >= "3.10" and python_version < "3.12"
-iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "3.12"
-jinja2==3.1.4 ; python_version >= "3.10" and python_version < "3.12"
 kiwisolver==1.4.5 ; python_version >= "3.10" and python_version < "3.12"
-markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "3.12"
 matplotlib==3.9.0 ; python_version >= "3.10" and python_version < "3.12"
-mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "3.12"
 numpy==1.24.4 ; python_version >= "3.10" and python_version < "3.12"
 packaging==24.1 ; python_version >= "3.10" and python_version < "3.12"
 pandas==2.2.2 ; python_version >= "3.10" and python_version < "3.12"
-pathspec==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
 pillow==10.3.0 ; python_version >= "3.10" and python_version < "3.12"
-platformdirs==4.2.2 ; python_version >= "3.10" and python_version < "3.12"
 plotly==5.22.0 ; python_version >= "3.10" and python_version < "3.12"
-pluggy==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 protobuf==4.25.3 ; python_version >= "3.10" and python_version < "3.12"
 protoc-gen-openapiv2==0.0.1 ; python_version >= "3.10" and python_version < "3.12"
 protofuzz @ git+https://github.com/trailofbits/protofuzz@a4fd0936ae501dba0fc89d3af4ead8ef3aaad1d1 ; python_version >= "3.10" and python_version < "3.12"
 psutil==5.9.8 ; python_version >= "3.10" and python_version < "3.12"
 pycparser==2.22 ; python_version >= "3.10" and python_version < "3.12"
 pynacl==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
-pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.12"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "3.12"
 python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "3.12"
 pytz==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 pywin32==306 ; python_version >= "3.10" and python_version < "3.12" and sys_platform == "win32"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.12"
 requests==2.32.3 ; python_version >= "3.10" and python_version < "3.12"
 scipy==1.13.1 ; python_version >= "3.10" and python_version < "3.12"
 setuptools==70.0.0 ; python_version >= "3.10" and python_version < "3.12"
 six==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
 tenacity==8.3.0 ; python_version >= "3.10" and python_version < "3.12"
 toml==0.10.2 ; python_version >= "3.10" and python_version < "3.12"
-tomli==2.0.1 ; python_version >= "3.10" and python_version < "3.11"
-typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.11"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
-vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@19ccb1c97e5f1d7f2676e89fbb7a814d6172f311 ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 wrapt==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
diff --git a/tests/integration/test_amms.py b/tests/integration/test_amms.py
@@ -9,7 +9,7 @@
     create_and_faucet_wallet,
 )
 
-import vega_python_protos as protos
+import vega_protos as protos
 
 
 def next_epoch(vega: VegaServiceNull):

diff --git a/tests/integration/test_governance.py b/tests/integration/test_governance.py
@@ -1,6 +1,6 @@
 import pytest
 
-import vega_python_protos.protos.vega as vega_protos
+import vega_protos.protos.vega as vega_protos
 
 from vega_sim.null_service import VegaServiceNull
 from tests.integration.utils.fixtures import WalletConfig, vega_service

diff --git a/tests/integration/test_liquidation.py b/tests/integration/test_liquidation.py
@@ -1,5 +1,5 @@
 import pytest
-from vega_python_protos import protos
+from vega_protos import protos
 from vega_sim.null_service import VegaServiceNull
 from tests.integration.utils.fixtures import (
     ASSET_NAME,

diff --git a/tests/integration/test_trading.py b/tests/integration/test_trading.py
@@ -1,5 +1,5 @@
 import pytest
-import vega_python_protos.protos.vega as vega_protos
+import vega_protos.protos.vega as vega_protos
 from vega_sim.api.market import MarketConfig
 import vega_sim.builders as build
 

diff --git a/tests/integration/test_update_market.py b/tests/integration/test_update_market.py
@@ -9,9 +9,9 @@
 )
 from vega_sim.null_service import VegaServiceNull
 from vega_sim.service import MarketStateUpdateType
-import vega_python_protos.protos.vega as vega_protos
-import vega_python_protos.protos.vega.data.v1 as oracles_protos
-import vega_python_protos.protos.vega.data_source_pb2 as data_source_protos
+import vega_protos.protos.vega as vega_protos
+import vega_protos.protos.vega.data.v1 as oracles_protos
+import vega_protos.protos.vega.data_source_pb2 as data_source_protos
 
 
 LIQ = WalletConfig("liq", "liq")

diff --git a/tests/vega_protos/__init__.py b/tests/vega_protos/__init__.py
diff --git a/tests/vega_protos/test_protos.py b/tests/vega_protos/test_protos.py
@@ -0,0 +1,19 @@
+import pytest
+
+import vega_protos as protos
+
+
+@pytest.mark.vega_protos
+def test_data_node_attributes():
+    protos.data_node.api.v2.trading_data
+
+
+@pytest.mark.vega_protos
+def test_vega_attributes():
+    protos.vega.assets
+    protos.vega.chain_events
+    protos.vega.data_source
+    protos.vega.governance
+    protos.vega.markets
+    protos.vega.oracle
+    protos.vega.vega
diff --git a/tests/vega_query/__init__.py b/tests/vega_query/__init__.py
@@ -0,0 +1,6 @@
+import yaml
+import logging.config
+
+with open("vega_query/logging.yaml", "r") as file:
+    yaml_dict = yaml.safe_load(file)
+logging.config.dictConfig(yaml_dict)