From 31f42fffb4dafac902b243e043de783ac5af6112 Mon Sep 17 00:00:00 2001 From: Charlie <99198652+cdummett@users.noreply.github.com> Date: Wed, 28 Aug 2024 14:27:10 +0100 Subject: [PATCH] feat: nebula fuzzing (#693) * chore: remove print * feat: improve fuzzing configurability * chore: align benchmark run parser * chore: reduce volatility and auction frequency * chore: bump most recent version to check * fix: update branch fuzz tests --- scripts/check_tagged_versions.py | 2 +- tests/integration/test_fuzz.py | 2 +- vega_sim/api/governance.py | 1 - vega_sim/scenario/benchmark/run.py | 9 +- vega_sim/scenario/fuzzing/registry.py | 73 ++++++- vega_sim/scenario/fuzzing/run.py | 11 +- vega_sim/scenario/fuzzing/scenario.py | 267 ++++++++++++++------------ 7 files changed, 233 insertions(+), 132 deletions(-) diff --git a/scripts/check_tagged_versions.py b/scripts/check_tagged_versions.py index 7e5a4a41f..b1f8ea2d8 100644 --- a/scripts/check_tagged_versions.py +++ b/scripts/check_tagged_versions.py @@ -1,6 +1,6 @@ import requests -EARLIEST_VERSION_TO_CHECK = "v0.73.8" +EARLIEST_VERSION_TO_CHECK = "v0.78.0-preview.4" def check_versions(): diff --git a/tests/integration/test_fuzz.py b/tests/integration/test_fuzz.py index e42a97c18..d80fed825 100644 --- a/tests/integration/test_fuzz.py +++ b/tests/integration/test_fuzz.py @@ -9,7 +9,7 @@ def test_fuzz_run(): # Simply testing that it doesn't error with tempfile.TemporaryDirectory() as temp_dir: - scenario = REGISTRY["overnight"] + scenario = REGISTRY["research"] scenario.num_steps = 20 _run( scenario=scenario, diff --git a/vega_sim/api/governance.py b/vega_sim/api/governance.py index b6d0c3129..dda736b57 100644 --- a/vega_sim/api/governance.py +++ b/vega_sim/api/governance.py @@ -1201,7 +1201,6 @@ def update_volume_rebate_program( proposal.terms.update_volume_rebate_program.CopyFrom( vega_protos.governance.UpdateVolumeRebateProgram(changes=volume_rebate_program) ) - print(proposal) return _make_and_wait_for_proposal( wallet_name=wallet_name, wallet=wallet, diff --git a/vega_sim/scenario/benchmark/run.py b/vega_sim/scenario/benchmark/run.py index 051cec15b..3646892bc 100644 --- a/vega_sim/scenario/benchmark/run.py +++ b/vega_sim/scenario/benchmark/run.py @@ -146,7 +146,7 @@ def _run( if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("-m", "--market", required=True, type=str) + parser.add_argument("--scenario", required=True, type=str) parser.add_argument("-s", "--steps", default=600, type=int) parser.add_argument("-p", "--pause", action="store_true") parser.add_argument("-d", "--debug", action="store_true") @@ -162,12 +162,13 @@ def _run( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - if args.market not in REGISTRY: + if args.scenario not in REGISTRY: raise ValueError(f"Market {args.market} not found") - scenario = REGISTRY[args.market].num_steps = args.steps + scenario = REGISTRY[args.scenario] + scenario.num_steps = args.steps _run( - scenario=REGISTRY[args.market], + scenario=scenario, wallet=args.wallet, console=args.console, pause=args.pause, diff --git a/vega_sim/scenario/fuzzing/registry.py b/vega_sim/scenario/fuzzing/registry.py index b0e3bc3e4..57c1af3b2 100644 --- a/vega_sim/scenario/fuzzing/registry.py +++ b/vega_sim/scenario/fuzzing/registry.py @@ -4,7 +4,7 @@ REGISTRY = { - "overnight": FuzzingScenario( + "research": FuzzingScenario( block_length_seconds=1, step_length_seconds=30, benchmark_configs=[ @@ -64,5 +64,76 @@ ), ], initial_network_parameters={"validators.epoch.length": "10m"}, + fuzz_lps=True, + fuzz_amms=True, + fuzz_traders=True, + fuzz_rewards=True, + fuzz_rebates=True, + fuzz_discounts=True, + fuzz_referrals=True, + ), + "nebula": FuzzingScenario( + block_length_seconds=1, + step_length_seconds=30, + benchmark_configs=[ + BenchmarkConfig( + market_config=configs.mainnet.BTCUSDT.CONFIG, + initial_price=80000, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + BenchmarkConfig( + market_config=configs.mainnet.ETHUSDT.CONFIG, + initial_price=4000, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + BenchmarkConfig( + market_config=configs.mainnet.SOLUSDT.CONFIG, + initial_price=20, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + ], + initial_network_parameters={"validators.epoch.length": "10m"}, + fuzz_lps=True, + fuzz_amms=False, + fuzz_traders=True, + fuzz_rewards=True, + fuzz_rebates=True, + fuzz_discounts=True, + fuzz_referrals=True, + ), + "nebula-amm": FuzzingScenario( + block_length_seconds=1, + step_length_seconds=30, + benchmark_configs=[ + BenchmarkConfig( + market_config=configs.mainnet.BTCUSDT.CONFIG, + initial_price=80000, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + BenchmarkConfig( + market_config=configs.mainnet.ETHUSDT.CONFIG, + initial_price=4000, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + BenchmarkConfig( + market_config=configs.mainnet.SOLUSDT.CONFIG, + initial_price=20, + annualised_volatility=1.5, + notional_trade_volume=100, + ), + ], + initial_network_parameters={"validators.epoch.length": "10m"}, + fuzz_lps=True, + fuzz_amms=True, + fuzz_traders=True, + fuzz_rewards=True, + fuzz_rebates=True, + fuzz_discounts=True, + fuzz_referrals=True, ), } diff --git a/vega_sim/scenario/fuzzing/run.py b/vega_sim/scenario/fuzzing/run.py index 1d3b85f3c..08074d66b 100644 --- a/vega_sim/scenario/fuzzing/run.py +++ b/vega_sim/scenario/fuzzing/run.py @@ -149,7 +149,7 @@ def _run( if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("-m", "--market", required=True, type=str) + parser.add_argument("--scenario", required=True, type=str) parser.add_argument("-s", "--steps", default=600, type=int) parser.add_argument("-p", "--pause", action="store_true") parser.add_argument("-d", "--debug", action="store_true") @@ -165,12 +165,13 @@ def _run( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - if args.market not in REGISTRY: - raise ValueError(f"Market {args.market} not found") - scenario = REGISTRY[args.market].num_steps = args.steps + if args.scenario not in REGISTRY: + raise ValueError(f"Market {args.scenario} not found") + scenario = REGISTRY[args.scenario] + scenario.num_steps = args.steps _run( - scenario=REGISTRY[args.market], + scenario=scenario, wallet=args.wallet, console=args.console, pause=args.pause, diff --git a/vega_sim/scenario/fuzzing/scenario.py b/vega_sim/scenario/fuzzing/scenario.py index b1ec7f862..67f561bb4 100644 --- a/vega_sim/scenario/fuzzing/scenario.py +++ b/vega_sim/scenario/fuzzing/scenario.py @@ -36,6 +36,13 @@ def __init__( step_length_seconds: Optional[float] = None, output: bool = True, initial_network_parameters: Optional[Dict[str, Any]] = None, + fuzz_lps: bool = False, + fuzz_amms: bool = False, + fuzz_traders: bool = False, + fuzz_rewards: bool = False, + fuzz_rebates: bool = False, + fuzz_referrals: bool = False, + fuzz_discounts: bool = False, ): super().__init__( benchmark_configs=benchmark_configs, @@ -47,6 +54,19 @@ def __init__( initial_network_parameters=initial_network_parameters, ) + # Party fuzzing options + self.__fuzz_lps = fuzz_lps + self.__fuzz_amms = fuzz_amms + self.__fuzz_traders = fuzz_traders + + # Program fuzzing options + self.__fuzz_rebates = fuzz_rebates + self.__fuzz_referrals = fuzz_referrals + self.__fuzz_discounts = fuzz_discounts + + # Reward fuzzing options + self.__fuzz_rewards = fuzz_rewards + def configure_agents( self, vega: VegaServiceNull, @@ -61,116 +81,124 @@ def configure_agents( extra_agents = [] # Add a fuzzed volume discount program - extra_agents.append( - FuzzyVolumeDiscountProgramManager( - wallet_name="FuzzyVolumeDiscountProgramManager", - key_name=f"FuzzyVolumeDiscountProgramManager", - step_bias=0.01, - attempts_per_step=10, + if self.__fuzz_discounts: + extra_agents.append( + FuzzyVolumeDiscountProgramManager( + wallet_name="FuzzyVolumeDiscountProgramManager", + key_name=f"FuzzyVolumeDiscountProgramManager", + step_bias=0.01, + attempts_per_step=10, + ) ) - ) # Add a fuzzed referral program - extra_agents.append( - FuzzyVolumeRebateProgramManager( - wallet_name="FuzzyReferralProgramManager", - key_name=f"FuzzyReferralProgramManager", - step_bias=0.01, - attempts_per_step=10, + if self.__fuzz_rebates: + extra_agents.append( + FuzzyVolumeRebateProgramManager( + wallet_name="FuzzyReferralProgramManager", + key_name=f"FuzzyReferralProgramManager", + step_bias=0.01, + attempts_per_step=10, + ) ) - ) # Add a fuzzed referral program - extra_agents.append( - FuzzyReferralProgramManager( - wallet_name="FuzzyReferralProgramManager", - key_name=f"FuzzyReferralProgramManager", - step_bias=0.01, - attempts_per_step=10, + if self.__fuzz_referrals: + extra_agents.append( + FuzzyReferralProgramManager( + wallet_name="FuzzyReferralProgramManager", + key_name=f"FuzzyReferralProgramManager", + step_bias=0.01, + attempts_per_step=10, + ) ) - ) # Add simple blank agents as team creators, team members will be added later - extra_agents.extend( - [ - ReferralAgentWrapper( - agent=StateAgentWithWallet( - wallet_name="ReferralAgentWrapper", - key_name=f"ReferralAgentWrapper_{str(i_agent).zfill(3)}", - tag=f"{str(i_agent).zfill(3)}", - ), - is_referrer=True, - team_name=f"teamName_{str(i_agent).zfill(3)}", - team_url=f"teamUrl_{str(i_agent).zfill(3)}", - avatar_url=f"avatarUrl_{str(i_agent).zfill(3)}", - closed=False, - ) - for i_agent in range(3) - ] - ) - - # For each market, add a fuzzing agent and a fuzzy liquidity provider to each - # team. Agents share keys across markets. - for benchmark_config in self.benchmark_configs: - market_name = benchmark_config.market_config.instrument.name - market_code = benchmark_config.market_config.instrument.code + if self.__fuzz_referrals: extra_agents.extend( [ ReferralAgentWrapper( - agent=FuzzingAgent( - wallet_name="FuzzingAgent", - key_name=( - f"FuzzingAgent_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" - ), - market_name=market_name, - settlement_asset_mint=benchmark_config.initial_price / 100, - quote_asset_mint=benchmark_config.initial_price / 100, - base_asset_mint=1 / 100, - tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", + agent=StateAgentWithWallet( + wallet_name="ReferralAgentWrapper", + key_name=f"ReferralAgentWrapper_{str(i_agent).zfill(3)}", + tag=f"{str(i_agent).zfill(3)}", ), - referrer_wallet_name="ReferralAgentWrapper", - referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", + is_referrer=True, + team_name=f"teamName_{str(i_agent).zfill(3)}", + team_url=f"teamUrl_{str(i_agent).zfill(3)}", + avatar_url=f"avatarUrl_{str(i_agent).zfill(3)}", + closed=False, ) - for i_referrer, i_agent in itertools.product(range(2), range(2)) + for i_agent in range(3) ] ) - extra_agents.extend( - [ - ReferralAgentWrapper( - agent=FuzzyLiquidityProvider( - wallet_name="FuzzyLiquidityProvider", - key_name=( - f"FuzzyLiquidityProvider_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" + + # For each market, add a fuzzing agent and a fuzzy liquidity provider to each + # team. Agents share keys across markets. + for benchmark_config in self.benchmark_configs: + market_name = benchmark_config.market_config.instrument.name + market_code = benchmark_config.market_config.instrument.code + if self.__fuzz_traders: + extra_agents.extend( + [ + ReferralAgentWrapper( + agent=FuzzingAgent( + wallet_name="FuzzingAgent", + key_name=( + f"FuzzingAgent_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" + ), + market_name=market_name, + settlement_asset_mint=benchmark_config.initial_price + / 100, + quote_asset_mint=benchmark_config.initial_price / 100, + base_asset_mint=1 / 100, + tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", ), - market_name=market_name, - initial_asset_mint=5_000, - tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", - ), - referrer_wallet_name="ReferralAgentWrapper", - referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", - ) - for i_referrer, i_agent in itertools.product(range(2), range(2)) - ] - ) - extra_agents.extend( - [ - ReferralAgentWrapper( - agent=FuzzedAutomatedMarketMaker( - wallet_name="FuzzedAutomatedMarketMaker", - key_name=( - f"FuzzedAutomatedMarketMaker{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" + referrer_wallet_name="ReferralAgentWrapper", + referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", + ) + for i_referrer, i_agent in itertools.product(range(2), range(2)) + ] + ) + if self.__fuzz_lps: + extra_agents.extend( + [ + ReferralAgentWrapper( + agent=FuzzyLiquidityProvider( + wallet_name="FuzzyLiquidityProvider", + key_name=( + f"FuzzyLiquidityProvider_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" + ), + market_name=market_name, + initial_asset_mint=5_000, + tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", ), - market_name=market_name, - submit_probability=0.5, - amend_probability=0.5, - cancel_probability=0.1, - initial_asset_mint=1e6, - tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", - ), - referrer_wallet_name="ReferralAgentWrapper", - referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", - ) - for i_referrer, i_agent in itertools.product(range(2), range(2)) - ] - ) + referrer_wallet_name="ReferralAgentWrapper", + referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", + ) + for i_referrer, i_agent in itertools.product(range(2), range(2)) + ] + ) + if self.__fuzz_amms: + extra_agents.extend( + [ + ReferralAgentWrapper( + agent=FuzzedAutomatedMarketMaker( + wallet_name="FuzzedAutomatedMarketMaker", + key_name=( + f"FuzzedAutomatedMarketMaker{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}" + ), + market_name=market_name, + submit_probability=0.5, + amend_probability=0.5, + cancel_probability=0.1, + initial_asset_mint=1e6, + tag=f"{market_code}_{str(i_referrer).zfill(3)}_{str(i_agent).zfill(3)}", + ), + referrer_wallet_name="ReferralAgentWrapper", + referrer_key_name=f"ReferralAgentWrapper_{str(i_referrer).zfill(3)}", + ) + for i_referrer, i_agent in itertools.product(range(2), range(2)) + ] + ) extra_agents.append( TransactionDelayChecker( wallet_name="TransactionDelayChecker", @@ -252,31 +280,32 @@ def configure_agents( ), ] ): - extra_agents.extend( - [ - RewardFunder( - wallet_name=f"RewardFunder", - key_name=f"RewardFunder_{asset_for_metric_name}_{str(i_agent).zfill(3)}", - reward_asset_name="VEGA", - account_type=account_type, - transfer_amount=100, - asset_for_metric_name=asset_for_metric_name, - metric=metric, - market_names=[market_name], - initial_mint=1e9, - entity_scope=entity_scope, - distribution_strategy=distribution_strategy, - tag=(f"{entity_scope}_{asset_for_metric_name}_{metric}"), - ) - for asset_for_metric_name, entity_scope in itertools.product( - list(asset_names), - [ - protos.vega.vega.ENTITY_SCOPE_INDIVIDUALS, - protos.vega.vega.ENTITY_SCOPE_TEAMS, - ], - ) - ] - ) + if self.__fuzz_rewards: + extra_agents.extend( + [ + RewardFunder( + wallet_name=f"RewardFunder", + key_name=f"RewardFunder_{asset_for_metric_name}_{str(i_agent).zfill(3)}", + reward_asset_name="VEGA", + account_type=account_type, + transfer_amount=100, + asset_for_metric_name=asset_for_metric_name, + metric=metric, + market_names=[market_name], + initial_mint=1e9, + entity_scope=entity_scope, + distribution_strategy=distribution_strategy, + tag=(f"{entity_scope}_{asset_for_metric_name}_{metric}"), + ) + for asset_for_metric_name, entity_scope in itertools.product( + list(asset_names), + [ + protos.vega.vega.ENTITY_SCOPE_INDIVIDUALS, + protos.vega.vega.ENTITY_SCOPE_TEAMS, + ], + ) + ] + ) extra_agents = {agent.name(): agent for agent in extra_agents} agents.update(extra_agents) return agents