From cc93ba4f79f56570678880464913822e8be14ca8 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 2 Jan 2025 15:53:44 +0200 Subject: [PATCH 01/24] First step- seprate api logic from report crafting --- Quorum/apis/governance/__init__.py | 0 Quorum/apis/governance/aave_governance.py | 58 +++++++++ Quorum/auto_report/aave_tags.py | 141 ++++++++++------------ 3 files changed, 122 insertions(+), 77 deletions(-) create mode 100644 Quorum/apis/governance/__init__.py create mode 100644 Quorum/apis/governance/aave_governance.py diff --git a/Quorum/apis/governance/__init__.py b/Quorum/apis/governance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py new file mode 100644 index 0000000..81c133b --- /dev/null +++ b/Quorum/apis/governance/aave_governance.py @@ -0,0 +1,58 @@ +import requests +from dataclasses import dataclass + +BASE_BGD_CACHE_REPO = 'https://raw.githubusercontent.com/bgd-labs/v3-governance-cache/refs/heads/main/cache' +PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' +BASE_SEATBELT_REPO = 'https://github.com/bgd-labs/seatbelt-gov-v3/blob/main/reports' +SEATBELT_PAYLOADS_URL = f'{BASE_SEATBELT_REPO}/payloads' + +@dataclass +class ChainInfo: + name: str + block_explorer_link: str + + +AAVE_CHAIN_MAPPING = { + '1': ChainInfo('Ethereum', 'https://etherscan.io/address'), + '137': ChainInfo('Polygon', 'https://polygonscan.com/address'), + '43114': ChainInfo('Avalanche', 'https://snowtrace.io/address'), + '8453': ChainInfo('Base', 'https://basescan.org/address'), + '42161': ChainInfo('Arbitrum One', 'https://arbiscan.io/address'), + '1088': ChainInfo('Metis', 'https://explorer.metis.io/address'), + '10': ChainInfo('OP Mainnet', 'https://optimistic.etherscan.io/address'), + '56': ChainInfo('BNB Smart Chain', 'https://bscscan.com/address'), + '100': ChainInfo('Gnosis', 'https://gnosisscan.io/address'), + '534352': ChainInfo('Scroll', 'https://scrollscan.com/address'), + '324': ChainInfo('zkSync Era', 'https://era.zksync.network/address'), + '59144': ChainInfo('Linea', 'https://lineascan.build/'), +} + + +class AaveGovernanceAPI: + """ + A utility class to interact with the BGD governance cache and retrieve + relevant information about Aave proposals and payload addresses. + """ + + def __init__(self) -> None: + self.session = requests.Session() + + def get_proposal_data(self, proposal_id: int) -> dict: + """ + Fetches the proposal data from the BGD governance cache. + """ + proposal_data_link = f'{PROPOSALS_URL}/{proposal_id}.json' + resp = self.session.get(proposal_data_link) + resp.raise_for_status() + return resp.json() + + def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) -> list[str]: + """ + Fetches and returns the addresses from a given chain/payload. + """ + url = f'{BASE_BGD_CACHE_REPO}/{chain_id}/{controller}/payloads/{payload_id}.json' + resp = self.session.get(url) + resp.raise_for_status() + + payload_data = resp.json() + return [a['target'] for a in payload_data['payload']['actions']] diff --git a/Quorum/auto_report/aave_tags.py b/Quorum/auto_report/aave_tags.py index 4a739af..79d8cb7 100644 --- a/Quorum/auto_report/aave_tags.py +++ b/Quorum/auto_report/aave_tags.py @@ -1,87 +1,74 @@ -import requests -from dataclasses import dataclass import json5 as json - -BASE_BGD_CACHE_REPO = 'https://raw.githubusercontent.com/bgd-labs/v3-governance-cache/refs/heads/main/cache' -PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' -BASE_SEATBELT_REPO = 'https://github.com/bgd-labs/seatbelt-gov-v3/blob/main/reports' -SEATBELT_PAYLOADS_URL = f'{BASE_SEATBELT_REPO}/payloads' - - -@dataclass -class ChainInfo: - name: str - block_explorer_link: str - - -AAVE_CHAIN_MAPPING = { - '1': ChainInfo('Ethereum', 'https://etherscan.io/address'), - '137': ChainInfo('Polygon', 'https://polygonscan.com/address'), - '43114': ChainInfo('Avalanche', 'https://snowtrace.io/address'), - '8453': ChainInfo('Base', 'https://basescan.org/address'), - '42161': ChainInfo('Arbitrum One', 'https://arbiscan.io/address'), - '1088': ChainInfo('Metis', 'https://explorer.metis.io/address'), - '10': ChainInfo('OP Mainnet', 'https://optimistic.etherscan.io/address'), - '56': ChainInfo('BNB Smart Chain', 'https://bscscan.com/address'), - '100': ChainInfo('Gnosis', 'https://gnosisscan.io/address'), - '534352': ChainInfo('Scroll', 'https://scrollscan.com/address'), - '324': ChainInfo('zkSync Era', 'https://era.zksync.network/address'), - "59144": ChainInfo('Linea', 'https://lineascan.build/'), -} - - -def __extract_payload_addresses(session: requests.Session, chain_id: str, controller: str, payload_id: int) -> list[str]: - resp = session.get(f'{BASE_BGD_CACHE_REPO}/{chain_id}/{controller}/payloads/{payload_id}.json') - resp.raise_for_status() - - payload_data = resp.json() - - return [a['target'] for a in payload_data['payload']['actions']] - +from Quorum.apis.governance.aave_governance import ( + AaveGovernanceAPI, + AAVE_CHAIN_MAPPING, + BASE_SEATBELT_REPO, + SEATBELT_PAYLOADS_URL, +) def get_aave_tags(proposal_id: int) -> dict: - with requests.Session() as session: - proposal_data_link = f'{PROPOSALS_URL}/{proposal_id}.json' - resp = session.get(proposal_data_link) - resp.raise_for_status() - - proposal_data: dict = resp.json() + """ + Utility function that orchestrates calls to AaveGovernanceAPI + and compiles the final dictionary of tags for a given proposal. + """ + api = AaveGovernanceAPI() + proposal_data = api.get_proposal_data(proposal_id) + + ipfs: dict = proposal_data.get('ipfs', {}) + proposal: dict = proposal_data.get('proposal', {}) + create_event: dict = proposal_data.get('events', [{}])[0] # The create event is always the first. + + tags = {} + tags['proposal_id'] = str(proposal_id) + tags['proposal_title'] = ipfs.get('title', 'N/A') + tags['voting_link'] = f'https://vote.onaave.com/proposal/?proposalId={proposal_id}' + tags['gov_forum_link'] = ipfs.get('discussions', 'N/A') + + # Prepare lists for multi-chain payload references + tags['chain'], tags['payload_link'], tags['payload_seatbelt_link'] = [], [], [] + + for p in proposal.get('payloads', []): + if not all(k in p for k in ['chain', 'payloadsController', 'payloadId']): + # Skip incomplete payload definitions + continue + + chain_id = p['chain'] + controller = p['payloadsController'] + pid = p['payloadId'] + + # Extract addresses from the payload + addresses = api.get_payload_addresses(chain_id, controller, pid) + + # For each address, add chain name, block explorer link, seatbelt link, etc. + for i, address in enumerate(addresses, 1): + chain_info = AAVE_CHAIN_MAPPING.get(chain_id) + if not chain_info: + # If chain info is missing, skip + continue - ipfs: dict = proposal_data.get('ipfs', {}) - proposal: dict = proposal_data.get('proposal', {}) - create_event: dict = proposal_data.get('events', [{}])[0] # The create event is always the first. + chain_display = chain_info.name + (f' {i}' if i != 1 else '') + tags['chain'].append(chain_display) + + block_explorer_link = f'{chain_info.block_explorer_link}/{address}' + tags['payload_link'].append(block_explorer_link) + + seatbelt_link = f'{SEATBELT_PAYLOADS_URL}/{chain_id}/{controller}/{pid}.md' + tags['payload_seatbelt_link'].append(seatbelt_link) - tags = {} - tags['proposal_id'] = str(proposal_id) - tags['proposal_title'] = ipfs.get('title', 'N/A') - tags['voting_link'] = f'https://vote.onaave.com/proposal/?proposalId={proposal_id}' - tags['gov_forum_link'] = ipfs.get('discussions', 'N/A') + tags['transaction_hash'] = create_event.get('transactionHash', 'N/A') + tags['transaction_link'] = f'https://etherscan.io/tx/{tags["transaction_hash"]}' - tags['chain'], tags['payload_link'], tags['payload_seatbelt_link'] = [], [], [] - for p in proposal.get('payloads', []): - # These are necessary fields in the payload data to construct the payload fields. - if not all(k in p for k in ['chain', 'payloadsController', 'payloadId']): - continue - addresses = __extract_payload_addresses(session, p['chain'], p['payloadsController'], p['payloadId']) - for i, address in enumerate(addresses, 1): - tags['chain'].append(AAVE_CHAIN_MAPPING[p['chain']].name + (f' {i}' if i != 1 else '')) - tags['payload_link'].append(f'{AAVE_CHAIN_MAPPING[p["chain"]].block_explorer_link}/{address}') - tags['payload_seatbelt_link'].append( - f'{SEATBELT_PAYLOADS_URL}/{p["chain"]}/{p["payloadsController"]}/{p["payloadId"]}.md' - ) - - tags['transaction_hash'] = create_event.get('transactionHash', 'N/A') - tags['transaction_link'] = f'https://etherscan.io/tx/{tags["transaction_hash"]}' + args: dict = create_event.get('args', {}) + tags['creator'] = args.get('creator', 'N/A') + tags['access_level'] = str(args.get('accessLevel', 'N/A')) + tags['ipfs_hash'] = args.get('ipfsHash', 'N/A') - args: dict = create_event.get('args', {}) - tags['creator'] = args.get('creator', 'N/A') - tags['access_level'] = str(args.get('accessLevel', 'N/A')) - tags['ipfs_hash'] = args.get('ipfsHash', 'N/A') - - tags['createProposal_parameters_data'] = json.dumps({k: proposal.get(k, 'N/A') for k - in ['payloads', 'votingPortal', 'ipfsHash']}, indent=4) + tags['createProposal_parameters_data'] = json.dumps( + {k: proposal.get(k, 'N/A') for k in ['payloads', 'votingPortal', 'ipfsHash']}, + indent=4 + ) - tags['seatbelt_link'] = f'{BASE_SEATBELT_REPO}/proposals/{proposal_id}.md' + tags['seatbelt_link'] = f'{BASE_SEATBELT_REPO}/proposals/{proposal_id}.md' - return tags + return tags From 5b2f399296ab747579c15dcf0c5a1310da84c427 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 2 Jan 2025 16:33:17 +0200 Subject: [PATCH 02/24] Move to data models + fixed tests --- Quorum/apis/governance/aave_governance.py | 101 ++++++++++++++---- Quorum/auto_report/aave_tags.py | 86 ++++++++------- .../tests/expected/test_auto_report/v3-132.md | 50 ++++----- 3 files changed, 152 insertions(+), 85 deletions(-) diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py index 81c133b..7efdd47 100644 --- a/Quorum/apis/governance/aave_governance.py +++ b/Quorum/apis/governance/aave_governance.py @@ -1,33 +1,91 @@ import requests -from dataclasses import dataclass +from typing import Optional, List +from pydantic import BaseModel, Field +import json5 as json +# ============================== +# Constants / Endpoints +# ============================== BASE_BGD_CACHE_REPO = 'https://raw.githubusercontent.com/bgd-labs/v3-governance-cache/refs/heads/main/cache' PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' BASE_SEATBELT_REPO = 'https://github.com/bgd-labs/seatbelt-gov-v3/blob/main/reports' SEATBELT_PAYLOADS_URL = f'{BASE_SEATBELT_REPO}/payloads' -@dataclass -class ChainInfo: + +# ============================== +# Chain Info Model +# ============================== +class ChainInfo(BaseModel): name: str block_explorer_link: str +# ============================== +# Data Models for BGD JSON +# ============================== +class IPFSData(BaseModel): + title: Optional[str] = None + discussions: Optional[str] = None + + +class PayloadData(BaseModel): + chain: str + payloads_controller: str = Field(alias='payloadsController') + payload_id: int = Field(alias='payloadId') + + class Config: + allow_population_by_alias = True + + +class ProposalData(BaseModel): + payloads: list[PayloadData] = Field(default_factory=list) + votingPortal: Optional[str] = None + ipfsHash: Optional[str] = None + + +class EventArgs(BaseModel): + creator: Optional[str] = None + accessLevel: Optional[int] = None + ipfsHash: Optional[str] = None + + +class EventData(BaseModel): + transactionHash: Optional[str] = None + args: EventArgs = Field(default_factory=EventArgs) + + +class BGDProposalData(BaseModel): + """ + Represents the entire JSON structure returned by the BGD cache + for a given proposal. + """ + ipfs: Optional[IPFSData] = None + proposal: Optional[ProposalData] = None + events: List[EventData] = Field(default_factory=list) + + +# ============================== +# Mapping for Chains +# ============================== AAVE_CHAIN_MAPPING = { - '1': ChainInfo('Ethereum', 'https://etherscan.io/address'), - '137': ChainInfo('Polygon', 'https://polygonscan.com/address'), - '43114': ChainInfo('Avalanche', 'https://snowtrace.io/address'), - '8453': ChainInfo('Base', 'https://basescan.org/address'), - '42161': ChainInfo('Arbitrum One', 'https://arbiscan.io/address'), - '1088': ChainInfo('Metis', 'https://explorer.metis.io/address'), - '10': ChainInfo('OP Mainnet', 'https://optimistic.etherscan.io/address'), - '56': ChainInfo('BNB Smart Chain', 'https://bscscan.com/address'), - '100': ChainInfo('Gnosis', 'https://gnosisscan.io/address'), - '534352': ChainInfo('Scroll', 'https://scrollscan.com/address'), - '324': ChainInfo('zkSync Era', 'https://era.zksync.network/address'), - '59144': ChainInfo('Linea', 'https://lineascan.build/'), + '1': ChainInfo(name='Ethereum', block_explorer_link='https://etherscan.io/address'), + '137': ChainInfo(name='Polygon', block_explorer_link='https://polygonscan.com/address'), + '43114': ChainInfo(name='Avalanche', block_explorer_link='https://snowtrace.io/address'), + '8453': ChainInfo(name='Base', block_explorer_link='https://basescan.org/address'), + '42161': ChainInfo(name='Arbitrum One', block_explorer_link='https://arbiscan.io/address'), + '1088': ChainInfo(name='Metis', block_explorer_link='https://explorer.metis.io/address'), + '10': ChainInfo(name='OP Mainnet', block_explorer_link='https://optimistic.etherscan.io/address'), + '56': ChainInfo(name='BNB Smart Chain',block_explorer_link='https://bscscan.com/address'), + '100': ChainInfo(name='Gnosis', block_explorer_link='https://gnosisscan.io/address'), + '534352':ChainInfo(name='Scroll', block_explorer_link='https://scrollscan.com/address'), + '324': ChainInfo(name='zkSync Era', block_explorer_link='https://era.zksync.network/address'), + '59144': ChainInfo(name='Linea', block_explorer_link='https://lineascan.build/') } +# ============================== +# AaveGovernanceAPI +# ============================== class AaveGovernanceAPI: """ A utility class to interact with the BGD governance cache and retrieve @@ -37,16 +95,20 @@ class AaveGovernanceAPI: def __init__(self) -> None: self.session = requests.Session() - def get_proposal_data(self, proposal_id: int) -> dict: + def get_proposal_data(self, proposal_id: int) -> BGDProposalData: """ - Fetches the proposal data from the BGD governance cache. + Fetches the proposal data from the BGD governance cache and + returns a pydantic-validated object. """ proposal_data_link = f'{PROPOSALS_URL}/{proposal_id}.json' resp = self.session.get(proposal_data_link) resp.raise_for_status() - return resp.json() - def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) -> list[str]: + raw_json = resp.json() + # Parse into our data model + return BGDProposalData(**raw_json) + + def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) -> List[str]: """ Fetches and returns the addresses from a given chain/payload. """ @@ -55,4 +117,5 @@ def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) resp.raise_for_status() payload_data = resp.json() + # We only need the 'target' field from each action return [a['target'] for a in payload_data['payload']['actions']] diff --git a/Quorum/auto_report/aave_tags.py b/Quorum/auto_report/aave_tags.py index 79d8cb7..83664e8 100644 --- a/Quorum/auto_report/aave_tags.py +++ b/Quorum/auto_report/aave_tags.py @@ -1,74 +1,88 @@ import json5 as json +from typing import Any, Dict +# Import the data models and API from Quorum.apis.governance.aave_governance import ( AaveGovernanceAPI, AAVE_CHAIN_MAPPING, BASE_SEATBELT_REPO, SEATBELT_PAYLOADS_URL, + BGDProposalData, + IPFSData, + ProposalData, + EventData, ) -def get_aave_tags(proposal_id: int) -> dict: + +def get_aave_tags(proposal_id: int) -> Dict[str, Any]: """ Utility function that orchestrates calls to AaveGovernanceAPI and compiles the final dictionary of tags for a given proposal. + + Returns: + A dictionary that can be directly rendered by your Jinja2 template. """ api = AaveGovernanceAPI() - proposal_data = api.get_proposal_data(proposal_id) + bgd_data: BGDProposalData = api.get_proposal_data(proposal_id) - ipfs: dict = proposal_data.get('ipfs', {}) - proposal: dict = proposal_data.get('proposal', {}) - create_event: dict = proposal_data.get('events', [{}])[0] # The create event is always the first. + # Safely unwrap fields (some might be None). + ipfs_data: IPFSData = bgd_data.ipfs or IPFSData() + proposal_data: ProposalData = bgd_data.proposal or ProposalData() + create_event: EventData = bgd_data.events[0] if bgd_data.events else EventData() - tags = {} + # Construct an empty dictionary for the Jinja2 context + tags: Dict[str, Any] = {} + + # Basic info tags['proposal_id'] = str(proposal_id) - tags['proposal_title'] = ipfs.get('title', 'N/A') + tags['proposal_title'] = ipfs_data.title if ipfs_data.title else 'N/A' tags['voting_link'] = f'https://vote.onaave.com/proposal/?proposalId={proposal_id}' - tags['gov_forum_link'] = ipfs.get('discussions', 'N/A') - - # Prepare lists for multi-chain payload references - tags['chain'], tags['payload_link'], tags['payload_seatbelt_link'] = [], [], [] + tags['gov_forum_link'] = ipfs_data.discussions if ipfs_data.discussions else 'N/A' - for p in proposal.get('payloads', []): - if not all(k in p for k in ['chain', 'payloadsController', 'payloadId']): - # Skip incomplete payload definitions - continue + # Multi-chain references + tags['chain'] = [] + tags['payload_link'] = [] + tags['payload_seatbelt_link'] = [] - chain_id = p['chain'] - controller = p['payloadsController'] - pid = p['payloadId'] + # Go through each payload in the proposal + for p in proposal_data.payloads: + # For each payload, retrieve the addresses from the API + addresses = api.get_payload_addresses( + chain_id = p.chain, + controller = p.payloads_controller, + payload_id = p.payload_id + ) - # Extract addresses from the payload - addresses = api.get_payload_addresses(chain_id, controller, pid) - - # For each address, add chain name, block explorer link, seatbelt link, etc. + # For each address, build up the chain/payload references for i, address in enumerate(addresses, 1): - chain_info = AAVE_CHAIN_MAPPING.get(chain_id) + chain_info = AAVE_CHAIN_MAPPING.get(p.chain) if not chain_info: # If chain info is missing, skip continue chain_display = chain_info.name + (f' {i}' if i != 1 else '') tags['chain'].append(chain_display) - + block_explorer_link = f'{chain_info.block_explorer_link}/{address}' tags['payload_link'].append(block_explorer_link) - - seatbelt_link = f'{SEATBELT_PAYLOADS_URL}/{chain_id}/{controller}/{pid}.md' + + seatbelt_link = f'{SEATBELT_PAYLOADS_URL}/{p.chain}/{p.payloads_controller}/{p.payload_id}.md' tags['payload_seatbelt_link'].append(seatbelt_link) - tags['transaction_hash'] = create_event.get('transactionHash', 'N/A') - tags['transaction_link'] = f'https://etherscan.io/tx/{tags["transaction_hash"]}' + # Transaction info + transaction_hash = create_event.transactionHash or 'N/A' + tags['transaction_hash'] = transaction_hash + tags['transaction_link'] = f'https://etherscan.io/tx/{transaction_hash}' - args: dict = create_event.get('args', {}) - tags['creator'] = args.get('creator', 'N/A') - tags['access_level'] = str(args.get('accessLevel', 'N/A')) - tags['ipfs_hash'] = args.get('ipfsHash', 'N/A') + # Creator + event args + args = create_event.args + tags['creator'] = args.creator if args.creator else 'N/A' + tags['access_level'] = str(args.accessLevel) if args.accessLevel is not None else 'N/A' + tags['ipfs_hash'] = args.ipfsHash if args.ipfsHash else 'N/A' - tags['createProposal_parameters_data'] = json.dumps( - {k: proposal.get(k, 'N/A') for k in ['payloads', 'votingPortal', 'ipfsHash']}, - indent=4 - ) + tags['createProposal_parameters_data'] = json.dumps(proposal_data.model_dump(), indent=4) + # seatbelt link for entire proposal tags['seatbelt_link'] = f'{BASE_SEATBELT_REPO}/proposals/{proposal_id}.md' return tags diff --git a/Quorum/tests/expected/test_auto_report/v3-132.md b/Quorum/tests/expected/test_auto_report/v3-132.md index 9cff55d..d5f95c9 100644 --- a/Quorum/tests/expected/test_auto_report/v3-132.md +++ b/Quorum/tests/expected/test_auto_report/v3-132.md @@ -56,63 +56,53 @@ Transaction: [0x423b2b381444d3a8a347536eaf643da3c7bc5e764ff4881863e012305d9542ba payloads: [ { chain: "1", - accessLevel: 1, - payloadsController: "0xdAbad81aF85554E9ae636395611C58F7eC1aAEc5", - payloadId: 146, + payloads_controller: "0xdAbad81aF85554E9ae636395611C58F7eC1aAEc5", + payload_id: 146, }, { chain: "137", - accessLevel: 1, - payloadsController: "0x401B5D0294E23637c18fcc38b1Bca814CDa2637C", - payloadId: 71, + payloads_controller: "0x401B5D0294E23637c18fcc38b1Bca814CDa2637C", + payload_id: 71, }, { chain: "43114", - accessLevel: 1, - payloadsController: "0x1140CB7CAfAcC745771C2Ea31e7B5C653c5d0B80", - payloadId: 42, + payloads_controller: "0x1140CB7CAfAcC745771C2Ea31e7B5C653c5d0B80", + payload_id: 42, }, { chain: "10", - accessLevel: 1, - payloadsController: "0x0E1a3Af1f9cC76A62eD31eDedca291E63632e7c4", - payloadId: 38, + payloads_controller: "0x0E1a3Af1f9cC76A62eD31eDedca291E63632e7c4", + payload_id: 38, }, { chain: "42161", - accessLevel: 1, - payloadsController: "0x89644CA1bB8064760312AE4F03ea41b05dA3637C", - payloadId: 39, + payloads_controller: "0x89644CA1bB8064760312AE4F03ea41b05dA3637C", + payload_id: 39, }, { chain: "1088", - accessLevel: 1, - payloadsController: "0x2233F8A66A728FBa6E1dC95570B25360D07D5524", - payloadId: 19, + payloads_controller: "0x2233F8A66A728FBa6E1dC95570B25360D07D5524", + payload_id: 19, }, { chain: "8453", - accessLevel: 1, - payloadsController: "0x2DC219E716793fb4b21548C0f009Ba3Af753ab01", - payloadId: 25, + payloads_controller: "0x2DC219E716793fb4b21548C0f009Ba3Af753ab01", + payload_id: 25, }, { chain: "100", - accessLevel: 1, - payloadsController: "0x9A1F491B86D09fC1484b5fab10041B189B60756b", - payloadId: 23, + payloads_controller: "0x9A1F491B86D09fC1484b5fab10041B189B60756b", + payload_id: 23, }, { chain: "534352", - accessLevel: 1, - payloadsController: "0x6b6B41c0f8C223715f712BE83ceC3c37bbfDC3fE", - payloadId: 15, + payloads_controller: "0x6b6B41c0f8C223715f712BE83ceC3c37bbfDC3fE", + payload_id: 15, }, { chain: "56", - accessLevel: 1, - payloadsController: "0xE5EF2Dd06755A97e975f7E282f828224F2C3e627", - payloadId: 17, + payloads_controller: "0xE5EF2Dd06755A97e975f7E282f828224F2C3e627", + payload_id: 17, }, ], votingPortal: "0x9b24C168d6A76b5459B1d47071a54962a4df36c3", From 59e6ddf9ca97f37a7f9cf5a9992521498cafcae1 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 2 Jan 2025 16:47:40 +0200 Subject: [PATCH 03/24] Extract all info for execution --- Quorum/apis/governance/aave_governance.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py index 7efdd47..c8c805c 100644 --- a/Quorum/apis/governance/aave_governance.py +++ b/Quorum/apis/governance/aave_governance.py @@ -64,6 +64,11 @@ class BGDProposalData(BaseModel): events: List[EventData] = Field(default_factory=list) +class PayloadAddresses(BaseModel): + chain: str + addresses: List[str] + + # ============================== # Mapping for Chains # ============================== @@ -119,3 +124,14 @@ def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) payload_data = resp.json() # We only need the 'target' field from each action return [a['target'] for a in payload_data['payload']['actions']] + + def get_all_payload_addresses(self, proposal_id: int) -> List[PayloadAddresses]: + """ + Retrieves a list of payload addresses for each chain in the proposal. + """ + data = self.get_proposal_data(proposal_id) + results = [] + for p in data.proposal.payloads: + addresses = self.get_payload_addresses(p.chain, p.payloads_controller, p.payload_id) + results.append(PayloadAddresses(chain=p.chain, addresses=addresses)) + return results From d82e269397b91059644f34ff1066fa2632910c77 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 5 Jan 2025 11:00:59 +0200 Subject: [PATCH 04/24] Organize --- Quorum/apis/governance/aave_governance.py | 120 +++++----------------- Quorum/apis/governance/data_models.py | 47 +++++++++ Quorum/auto_report/aave_tags.py | 39 +++++-- 3 files changed, 101 insertions(+), 105 deletions(-) create mode 100644 Quorum/apis/governance/data_models.py diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py index c8c805c..d272520 100644 --- a/Quorum/apis/governance/aave_governance.py +++ b/Quorum/apis/governance/aave_governance.py @@ -1,96 +1,9 @@ import requests -from typing import Optional, List -from pydantic import BaseModel, Field -import json5 as json +from Quorum.apis.governance.data_models import BGDProposalData, PayloadAddresses -# ============================== -# Constants / Endpoints -# ============================== BASE_BGD_CACHE_REPO = 'https://raw.githubusercontent.com/bgd-labs/v3-governance-cache/refs/heads/main/cache' PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' -BASE_SEATBELT_REPO = 'https://github.com/bgd-labs/seatbelt-gov-v3/blob/main/reports' -SEATBELT_PAYLOADS_URL = f'{BASE_SEATBELT_REPO}/payloads' - -# ============================== -# Chain Info Model -# ============================== -class ChainInfo(BaseModel): - name: str - block_explorer_link: str - - -# ============================== -# Data Models for BGD JSON -# ============================== -class IPFSData(BaseModel): - title: Optional[str] = None - discussions: Optional[str] = None - - -class PayloadData(BaseModel): - chain: str - payloads_controller: str = Field(alias='payloadsController') - payload_id: int = Field(alias='payloadId') - - class Config: - allow_population_by_alias = True - - -class ProposalData(BaseModel): - payloads: list[PayloadData] = Field(default_factory=list) - votingPortal: Optional[str] = None - ipfsHash: Optional[str] = None - - -class EventArgs(BaseModel): - creator: Optional[str] = None - accessLevel: Optional[int] = None - ipfsHash: Optional[str] = None - - -class EventData(BaseModel): - transactionHash: Optional[str] = None - args: EventArgs = Field(default_factory=EventArgs) - - -class BGDProposalData(BaseModel): - """ - Represents the entire JSON structure returned by the BGD cache - for a given proposal. - """ - ipfs: Optional[IPFSData] = None - proposal: Optional[ProposalData] = None - events: List[EventData] = Field(default_factory=list) - - -class PayloadAddresses(BaseModel): - chain: str - addresses: List[str] - - -# ============================== -# Mapping for Chains -# ============================== -AAVE_CHAIN_MAPPING = { - '1': ChainInfo(name='Ethereum', block_explorer_link='https://etherscan.io/address'), - '137': ChainInfo(name='Polygon', block_explorer_link='https://polygonscan.com/address'), - '43114': ChainInfo(name='Avalanche', block_explorer_link='https://snowtrace.io/address'), - '8453': ChainInfo(name='Base', block_explorer_link='https://basescan.org/address'), - '42161': ChainInfo(name='Arbitrum One', block_explorer_link='https://arbiscan.io/address'), - '1088': ChainInfo(name='Metis', block_explorer_link='https://explorer.metis.io/address'), - '10': ChainInfo(name='OP Mainnet', block_explorer_link='https://optimistic.etherscan.io/address'), - '56': ChainInfo(name='BNB Smart Chain',block_explorer_link='https://bscscan.com/address'), - '100': ChainInfo(name='Gnosis', block_explorer_link='https://gnosisscan.io/address'), - '534352':ChainInfo(name='Scroll', block_explorer_link='https://scrollscan.com/address'), - '324': ChainInfo(name='zkSync Era', block_explorer_link='https://era.zksync.network/address'), - '59144': ChainInfo(name='Linea', block_explorer_link='https://lineascan.build/') -} - - -# ============================== -# AaveGovernanceAPI -# ============================== class AaveGovernanceAPI: """ A utility class to interact with the BGD governance cache and retrieve @@ -102,8 +15,13 @@ def __init__(self) -> None: def get_proposal_data(self, proposal_id: int) -> BGDProposalData: """ - Fetches the proposal data from the BGD governance cache and - returns a pydantic-validated object. + Fetches and returns the data for a given proposal. + + Args: + proposal_id: The ID of the proposal to fetch. + + Returns: + A BGDProposalData object. """ proposal_data_link = f'{PROPOSALS_URL}/{proposal_id}.json' resp = self.session.get(proposal_data_link) @@ -113,9 +31,17 @@ def get_proposal_data(self, proposal_id: int) -> BGDProposalData: # Parse into our data model return BGDProposalData(**raw_json) - def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) -> List[str]: + def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) -> list[str]: """ - Fetches and returns the addresses from a given chain/payload. + Retrieves a list of payload addresses for a given payload ID, chain, and controller. + + Args: + chain_id: The chain ID for the proposal. + controller: The controller for the proposal. + payload_id: The ID of the payload to fetch. + + Returns: + A list of addresses that are part of the payload. """ url = f'{BASE_BGD_CACHE_REPO}/{chain_id}/{controller}/payloads/{payload_id}.json' resp = self.session.get(url) @@ -125,9 +51,15 @@ def get_payload_addresses(self, chain_id: str, controller: str, payload_id: int) # We only need the 'target' field from each action return [a['target'] for a in payload_data['payload']['actions']] - def get_all_payload_addresses(self, proposal_id: int) -> List[PayloadAddresses]: + def get_all_payloads_addresses(self, proposal_id: int) -> list[PayloadAddresses]: """ - Retrieves a list of payload addresses for each chain in the proposal. + Retrieves all payload addresses for a given proposal. + + Args: + proposal_id: The ID of the proposal to fetch. + + Returns: + A list of PayloadAddresses objects, each containing a chain ID and a list of addresses. """ data = self.get_proposal_data(proposal_id) results = [] diff --git a/Quorum/apis/governance/data_models.py b/Quorum/apis/governance/data_models.py new file mode 100644 index 0000000..6066933 --- /dev/null +++ b/Quorum/apis/governance/data_models.py @@ -0,0 +1,47 @@ +from typing import List, Optional +from pydantic import BaseModel, Field + +class IPFSData(BaseModel): + title: Optional[str] = None + discussions: Optional[str] = None + + +class PayloadData(BaseModel): + chain: str + payloads_controller: str = Field(alias='payloadsController') + payload_id: int = Field(alias='payloadId') + + class Config: + allow_population_by_alias = True + + +class ProposalData(BaseModel): + payloads: list[PayloadData] = Field(default_factory=list) + votingPortal: Optional[str] = None + ipfsHash: Optional[str] = None + + +class EventArgs(BaseModel): + creator: Optional[str] = None + accessLevel: Optional[int] = None + ipfsHash: Optional[str] = None + + +class EventData(BaseModel): + transactionHash: Optional[str] = None + args: EventArgs = Field(default_factory=EventArgs) + + +class BGDProposalData(BaseModel): + """ + Represents the entire JSON structure returned by the BGD cache + for a given proposal. + """ + ipfs: Optional[IPFSData] = None + proposal: Optional[ProposalData] = None + events: List[EventData] = Field(default_factory=list) + + +class PayloadAddresses(BaseModel): + chain: str + addresses: List[str] diff --git a/Quorum/auto_report/aave_tags.py b/Quorum/auto_report/aave_tags.py index 83664e8..586453d 100644 --- a/Quorum/auto_report/aave_tags.py +++ b/Quorum/auto_report/aave_tags.py @@ -1,17 +1,34 @@ import json5 as json +from pydantic import BaseModel from typing import Any, Dict -# Import the data models and API -from Quorum.apis.governance.aave_governance import ( - AaveGovernanceAPI, - AAVE_CHAIN_MAPPING, - BASE_SEATBELT_REPO, - SEATBELT_PAYLOADS_URL, - BGDProposalData, - IPFSData, - ProposalData, - EventData, -) +from Quorum.apis.governance.aave_governance import AaveGovernanceAPI +from Quorum.apis.governance.data_models import BGDProposalData, IPFSData, ProposalData, EventData + + +BASE_SEATBELT_REPO = 'https://github.com/bgd-labs/seatbelt-gov-v3/blob/main/reports' +SEATBELT_PAYLOADS_URL = f'{BASE_SEATBELT_REPO}/payloads' + + +class ChainInfo(BaseModel): + name: str + block_explorer_link: str + + +AAVE_CHAIN_MAPPING = { + '1': ChainInfo(name='Ethereum', block_explorer_link='https://etherscan.io/address'), + '137': ChainInfo(name='Polygon', block_explorer_link='https://polygonscan.com/address'), + '43114': ChainInfo(name='Avalanche', block_explorer_link='https://snowtrace.io/address'), + '8453': ChainInfo(name='Base', block_explorer_link='https://basescan.org/address'), + '42161': ChainInfo(name='Arbitrum One', block_explorer_link='https://arbiscan.io/address'), + '1088': ChainInfo(name='Metis', block_explorer_link='https://explorer.metis.io/address'), + '10': ChainInfo(name='OP Mainnet', block_explorer_link='https://optimistic.etherscan.io/address'), + '56': ChainInfo(name='BNB Smart Chain',block_explorer_link='https://bscscan.com/address'), + '100': ChainInfo(name='Gnosis', block_explorer_link='https://gnosisscan.io/address'), + '534352':ChainInfo(name='Scroll', block_explorer_link='https://scrollscan.com/address'), + '324': ChainInfo(name='zkSync Era', block_explorer_link='https://era.zksync.network/address'), + '59144': ChainInfo(name='Linea', block_explorer_link='https://lineascan.build/') +} def get_aave_tags(proposal_id: int) -> Dict[str, Any]: From f54e723dafeed08c355700dfd031088337719365 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 5 Jan 2025 11:08:01 +0200 Subject: [PATCH 05/24] Reverse mapping --- Quorum/apis/governance/aave_governance.py | 18 +++++++++++++++++- Quorum/apis/governance/data_models.py | 4 +++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py index d272520..29b9d40 100644 --- a/Quorum/apis/governance/aave_governance.py +++ b/Quorum/apis/governance/aave_governance.py @@ -1,9 +1,25 @@ import requests + +from Quorum.utils.chain_enum import Chain from Quorum.apis.governance.data_models import BGDProposalData, PayloadAddresses BASE_BGD_CACHE_REPO = 'https://raw.githubusercontent.com/bgd-labs/v3-governance-cache/refs/heads/main/cache' PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' +CHAIN_ID_TO_CHAIN = { + 1: Chain.ETH, + 42161: Chain.ARB, + 43114: Chain.AVAX, + 8453: Chain.BASE, + 56: Chain.BSC, + 100: Chain.GNO, + 10: Chain.OPT, + 137: Chain.POLY, + 534352: Chain.SCROLL, + 324: Chain.ZK, + 59144: Chain.LINEA, +} + class AaveGovernanceAPI: """ A utility class to interact with the BGD governance cache and retrieve @@ -65,5 +81,5 @@ def get_all_payloads_addresses(self, proposal_id: int) -> list[PayloadAddresses] results = [] for p in data.proposal.payloads: addresses = self.get_payload_addresses(p.chain, p.payloads_controller, p.payload_id) - results.append(PayloadAddresses(chain=p.chain, addresses=addresses)) + results.append(PayloadAddresses(chain=CHAIN_ID_TO_CHAIN[p.chain], addresses=addresses)) return results diff --git a/Quorum/apis/governance/data_models.py b/Quorum/apis/governance/data_models.py index 6066933..72536d2 100644 --- a/Quorum/apis/governance/data_models.py +++ b/Quorum/apis/governance/data_models.py @@ -1,6 +1,8 @@ from typing import List, Optional from pydantic import BaseModel, Field +from Quorum.utils.chain_enum import Chain + class IPFSData(BaseModel): title: Optional[str] = None discussions: Optional[str] = None @@ -43,5 +45,5 @@ class BGDProposalData(BaseModel): class PayloadAddresses(BaseModel): - chain: str + chain: Chain addresses: List[str] From a57b84bbbd0d7e71378146a672387df417f3c44b Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 5 Jan 2025 12:46:06 +0200 Subject: [PATCH 06/24] Add test for new entry points --- .github/workflows/CI.yml | 4 +- Quorum/apis/governance/aave_governance.py | 22 +- Quorum/checks/proposal_check.py | 52 +++++ Quorum/entry_points/check_proposal.py | 203 +++++++++---------- Quorum/entry_points/check_proposal_config.py | 111 ++++++++++ Quorum/entry_points/check_proposal_id.py | 135 ++++++++++++ Quorum/utils/arg_validations.py | 24 ++- setup.py | 2 + 8 files changed, 434 insertions(+), 119 deletions(-) create mode 100644 Quorum/checks/proposal_check.py create mode 100644 Quorum/entry_points/check_proposal_config.py create mode 100644 Quorum/entry_points/check_proposal_id.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d369ed3..5fb48e8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,4 +39,6 @@ jobs: - name: Execute Regression Tests run: | pytest Quorum/tests --maxfail=1 --disable-warnings --tb=short - CheckProposal --config Quorum/tests/regression.json + CheckProposal --customer Aave --chain Ethereum --proposal_address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637 + CheckProposalConfig --config Quorum/tests/regression.json + CheckProposalId --proposal_id 137 --customer Aave diff --git a/Quorum/apis/governance/aave_governance.py b/Quorum/apis/governance/aave_governance.py index 29b9d40..9ef7556 100644 --- a/Quorum/apis/governance/aave_governance.py +++ b/Quorum/apis/governance/aave_governance.py @@ -7,17 +7,17 @@ PROPOSALS_URL = f'{BASE_BGD_CACHE_REPO}/1/0x9AEE0B04504CeF83A65AC3f0e838D0593BCb2BC7/proposals' CHAIN_ID_TO_CHAIN = { - 1: Chain.ETH, - 42161: Chain.ARB, - 43114: Chain.AVAX, - 8453: Chain.BASE, - 56: Chain.BSC, - 100: Chain.GNO, - 10: Chain.OPT, - 137: Chain.POLY, - 534352: Chain.SCROLL, - 324: Chain.ZK, - 59144: Chain.LINEA, + '1': Chain.ETH, + '42161': Chain.ARB, + '43114': Chain.AVAX, + '8453': Chain.BASE, + '56': Chain.BSC, + '100': Chain.GNO, + '10': Chain.OPT, + '137': Chain.POLY, + '534352': Chain.SCROLL, + '324': Chain.ZK, + '59144': Chain.LINEA, } class AaveGovernanceAPI: diff --git a/Quorum/checks/proposal_check.py b/Quorum/checks/proposal_check.py new file mode 100644 index 0000000..cdb097d --- /dev/null +++ b/Quorum/checks/proposal_check.py @@ -0,0 +1,52 @@ +import Quorum.checks as Checks +import Quorum.utils.pretty_printer as pp +from Quorum.utils.chain_enum import Chain +from Quorum.apis.block_explorers.chains_api import ChainAPI +from Quorum.apis.price_feeds.price_feed_utils import PriceFeedProviderBase + + +def proposals_check(customer: str, chain: Chain, proposal_addresses: list[str], providers: list[PriceFeedProviderBase]) -> None: + """ + Check and compare source code files for given proposals. + + This function handles the main logic of fetching source code from the remote repository. + + Args: + customer (str): The customer name or identifier. + chain_name (str): The blockchain chain name. + proposal_addresses (list[str]): List of proposal addresses. + providers (list[PriceFeedProviderInterface]): List of price feed providers. + """ + api = ChainAPI(chain) + + pp.pretty_print(f"Processing customer {customer}, for chain: {chain}", pp.Colors.INFO) + for proposal_address in proposal_addresses: + pp.pretty_print(f"Processing proposal {proposal_address}", pp.Colors.INFO) + + try: + source_codes = api.get_source_code(proposal_address) + except ValueError as e: + error_message = ( + f"Payload address {proposal_address} is not verified on {chain.name} explorer.\n" + "We do not recommend to approve this proposal until the code is approved!\n" + "Try contacting the proposer and ask them to verify the contract.\n" + "No further checks are being performed on this payload." + ) + pp.pretty_print(error_message, pp.Colors.FAILURE) + # Skip further checks for this proposal + continue + + # Diff check + missing_files = Checks.DiffCheck(customer, chain, proposal_address, source_codes).find_diffs() + + # Review diff check + Checks.ReviewDiffCheck(customer, chain, proposal_address, missing_files).find_diffs() + + # Global variables check + Checks.GlobalVariableCheck(customer, chain, proposal_address, missing_files).check_global_variables() + + # Feed price check + Checks.PriceFeedCheck(customer, chain, proposal_address, missing_files, providers).verify_price_feed() + + # New listing check + Checks.NewListingCheck(customer, chain, proposal_address, missing_files).new_listing_check() diff --git a/Quorum/entry_points/check_proposal.py b/Quorum/entry_points/check_proposal.py index 3cc0f59..e450762 100644 --- a/Quorum/entry_points/check_proposal.py +++ b/Quorum/entry_points/check_proposal.py @@ -1,129 +1,120 @@ +""" +Quorum Proposal Checker + +This script is designed to fetch and compare smart contract source code for a specific proposal address +on a chosen blockchain chain for a designated customer. The workflow includes: + +1. Parsing command-line arguments to obtain the customer identifier, blockchain chain, and proposal address. +2. Loading the ground truth configuration for the specified customer. +3. Cloning or updating necessary Git repositories based on the configuration. +4. Executing a series of checks on the proposal's smart contract source code to ensure integrity and compliance. + +Usage: + python check_proposal.py --customer --chain --proposal_address + +Example: + python check_proposal.py --customer Aave --chain Ethereum --proposal_address 0x1234567890abcdef1234567890abcdef12345678 +""" + import argparse -from json import JSONDecodeError -import json5 as json -from typing import Any, Optional +from typing import Tuple from Quorum.utils.chain_enum import Chain from Quorum.apis.git_api.git_manager import GitManager -from Quorum.apis.price_feeds.price_feed_utils import PriceFeedProviderBase -from Quorum.apis.block_explorers.chains_api import ChainAPI -import Quorum.checks as Checks -import Quorum.utils.pretty_printer as pp +from Quorum.checks.proposal_check import proposals_check import Quorum.utils.config_loader as ConfigLoader import Quorum.utils.arg_validations as arg_valid -def parse_args() -> tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: - """ - Parse command line arguments for JSON configuration file or individual task parameters. - - Returns: - tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: - A tuple containing the path to the JSON configuration file, customer name, chain name, and proposal address. - """ - parser = argparse.ArgumentParser(description="Fetch and compare smart contract source code.") - parser.add_argument('--config', type=load_config, help="Path to JSON configuration file.") - parser.add_argument('--customer', type=str, help="Customer name or identifier.") - parser.add_argument('--chain', type=Chain, choices=[chain.value for chain in Chain], help="Blockchain chain.") - parser.add_argument('--proposal_address', type=arg_valid.validate_address, help="Ethereum proposal address.") - args = parser.parse_args() - - return args.config, args.customer, args.chain, args.proposal_address - - -def load_config(config_path: str) -> dict[str, Any] | None: +def parse_args() -> Tuple[str, Chain, str]: """ - Load and parse the JSON configuration file. + Parses command-line arguments required for executing proposal checks. - Args: - config_path (str): Path to the JSON configuration file. + This function utilizes Python's argparse module to define and parse the necessary command-line + arguments: + --customer: Name or identifier of the customer. + --chain: Blockchain chain to target, must be one of the defined Chain enum values. + --proposal_address: Ethereum proposal address, validated using `arg_valid.validate_address`. Returns: - dict[str, Any]: Parsed JSON data. - """ - try: - with open(config_path, 'r') as file: - config_data = json.load(file) - return config_data - except (FileNotFoundError, JSONDecodeError) as e: - pp.pretty_print(f"Failed to parse given config file {config_path}:\n{e}", pp.Colors.FAILURE) + A tuple containing: + customer (str): Customer name or identifier. + chain (Chain): Selected blockchain chain. + proposal_address (str): Validated Ethereum proposal address. - -def proposals_check(customer: str, chain: Chain, proposal_addresses: list[str], providers: list[PriceFeedProviderBase]) -> None: + Raises: + argparse.ArgumentError: If required arguments are missing or invalid. """ - Check and compare source code files for given proposals. + parser = argparse.ArgumentParser( + description="Fetch and compare smart contract source code for a given proposal." + ) + parser.add_argument( + '--customer', + type=str, + required=True, + help="Customer name or identifier." + ) + parser.add_argument( + '--chain', + type=Chain, + choices=list(Chain), + required=True, + help="Blockchain chain to target." + ) + parser.add_argument( + '--proposal_address', + type=arg_valid.validate_address, + required=True, + help="Ethereum proposal address (e.g., 0x...)." + ) + args = parser.parse_args() - This function handles the main logic of fetching source code from the remote repository. + return args.customer, args.chain, args.proposal_address - Args: - customer (str): The customer name or identifier. - chain_name (str): The blockchain chain name. - proposal_addresses (list[str]): List of proposal addresses. - providers (list[PriceFeedProviderInterface]): List of price feed providers. - """ - api = ChainAPI(chain) - - pp.pretty_print(f"Processing customer {customer}, for chain: {chain}", pp.Colors.INFO) - for proposal_address in proposal_addresses: - pp.pretty_print(f"Processing proposal {proposal_address}", pp.Colors.INFO) - - try: - source_codes = api.get_source_code(proposal_address) - except ValueError as e: - error_message = ( - f"Payload address {proposal_address} is not verified on {chain.name} explorer.\n" - "We do not recommend to approve this proposal until the code is approved!\n" - "Try contacting the proposer and ask them to verify the contract.\n" - "No further checks are being performed on this payload." - ) - pp.pretty_print(error_message, pp.Colors.FAILURE) - # Skip further checks for this proposal - continue - - # Diff check - missing_files = Checks.DiffCheck(customer, chain, proposal_address, source_codes).find_diffs() - - # Review diff check - Checks.ReviewDiffCheck(customer, chain, proposal_address, missing_files).find_diffs() - - # Global variables check - Checks.GlobalVariableCheck(customer, chain, proposal_address, missing_files).check_global_variables() - - # Feed price check - Checks.PriceFeedCheck(customer, chain, proposal_address, missing_files, providers).verify_price_feed() - - # New listing check - Checks.NewListingCheck(customer, chain, proposal_address, missing_files).new_listing_check() def main() -> None: """ - Main function to execute tasks based on command line arguments or JSON configuration. - - This function determines whether to run in single-task mode using command line arguments - or multi-task mode using a JSON configuration file. + Main execution function that orchestrates fetching, cloning/updating repositories, + and performing proposal checks based on the provided command-line arguments. + + Workflow: + 1. Parse command-line arguments to retrieve customer, chain, and proposal address. + 2. Load the ground truth configuration for the specified customer. + 3. Initialize and update Git repositories as per the configuration. + 4. Execute the proposal checks for the provided proposal address using the loaded configuration. + + Raises: + ValueError: If any of the required arguments (customer, chain, proposal_address) are missing. + FileNotFoundError: If the customer configuration file does not exist. + requests.HTTPError: If any of the HTTP requests to fetch data fail. """ - config_data, customer, chain, proposal_address = parse_args() - - if config_data: - # Multi-task mode using JSON configuration - for customer, chain_info in config_data.items(): - ground_truth_config = ConfigLoader.load_customer_config(customer) - GitManager(customer, ground_truth_config).clone_or_update() - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - for chain, proposals in chain_info.items(): - # Validate chain is supported by cast to Chain enum - chain = Chain(chain) - if proposals["Proposals"]: - proposals_check(customer, chain, proposals["Proposals"], price_feed_providers) - else: - # Single-task mode using command line arguments - if not (customer and chain and proposal_address): - raise ValueError("Customer, chain, and proposal_address must be specified if not using a config file.") - - ground_truth_config = ConfigLoader.load_customer_config(customer) - GitManager(customer, ground_truth_config).clone_or_update() - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - proposals_check(customer, chain, [proposal_address], price_feed_providers) + # Parse command-line arguments + customer, chain, proposal_address = parse_args() + + # Ensure all required arguments are provided + if not (customer and chain and proposal_address): + raise ValueError( + "Customer, chain, and proposal_address must be specified." + " Provide all three arguments when not using a config file." + ) + + # Load the customer's ground truth configuration + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Execute proposal checks + proposals_check( + customer=customer, + chain=chain, + proposal_addresses=[proposal_address], + providers=price_feed_providers + ) if __name__ == "__main__": diff --git a/Quorum/entry_points/check_proposal_config.py b/Quorum/entry_points/check_proposal_config.py new file mode 100644 index 0000000..6e31d75 --- /dev/null +++ b/Quorum/entry_points/check_proposal_config.py @@ -0,0 +1,111 @@ +""" +Quorum JSON Configuration Proposal Analyzer + +This script is designed to fetch and compare smart contract source code based on a JSON configuration file +for various customers and their respective blockchain chains. The workflow includes: + +1. Parsing command-line arguments to obtain the path to a JSON configuration file. +2. Loading the ground truth configuration for each specified customer. +3. Cloning or updating necessary Git repositories based on the configuration. +4. Executing a series of proposal checks on the provided proposal addresses to ensure integrity and compliance. + +Usage: + python check_proposal_config.py --config + +Example: + python check_proposal_config.py --config execution.json +""" + +import argparse +from typing import Dict, Any + +from Quorum.apis.git_api.git_manager import GitManager +from Quorum.checks.proposal_check import proposals_check +import Quorum.utils.config_loader as ConfigLoader +import Quorum.utils.arg_validations as arg_valid +from Quorum.utils.chain_enum import Chain + + +def parse_args() -> Dict[str, Any]: + """ + Parses command-line arguments required for executing proposal analysis based on a JSON configuration file. + + This function utilizes Python's argparse module to define and parse the necessary command-line + arguments: + --config: Path to the JSON configuration file containing customer and proposal details. + + Returns: + dict: Parsed JSON data from the configuration file. + + Raises: + argparse.ArgumentError: If the provided configuration file path is invalid or the file cannot be loaded. + """ + parser = argparse.ArgumentParser( + description="Fetch and compare smart contract source code based on a JSON configuration file." + ) + parser.add_argument( + '--config', + type=arg_valid.load_config, + required=True, + help="Path to the JSON configuration file." + ) + args = parser.parse_args() + + return args.config + + +def main() -> None: + """ + Main execution function that orchestrates fetching, cloning/updating repositories, + and performing proposal checks based on the provided JSON configuration file. + + Workflow: + 1. Parse command-line arguments to retrieve the configuration data. + 2. Iterate over each customer and their associated chain and proposal information. + 3. Load the ground truth configuration for each customer. + 4. Initialize GitManager with customer and configuration, then clone or update repositories. + 5. Retrieve price feed providers from the configuration. + 6. Execute proposal checks for each set of proposal addresses on the specified chains. + + Raises: + ValueError: If the specified customer is not supported. + FileNotFoundError: If the customer configuration file does not exist. + requests.HTTPError: If any of the HTTP requests to fetch data fail. + """ + # Parse command-line arguments to get the configuration data + config_data: Dict[str, Any] = parse_args() + + # Iterate over each customer and their respective chain and proposal information in the config + for customer, chain_info in config_data.items(): + # Load the ground truth configuration for the specified customer + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Iterate over each blockchain chain and its associated proposals for the customer + for chain, proposals in chain_info.items(): + try: + # Validate and convert the chain identifier to the Chain enum + chain_enum = Chain(chain) + except ValueError as e: + # Handle unsupported or invalid chain identifiers + raise ValueError(f"Unsupported or invalid chain '{chain}' for customer '{customer}': {e}") + + # Check if there are any proposals to process for the current chain + if proposals.get("Proposals"): + # Execute proposal checks for the specified customer, chain, and proposal addresses + proposals_check( + customer=customer, + chain=chain_enum, + proposal_addresses=proposals["Proposals"], + providers=price_feed_providers + ) + + +if __name__ == "__main__": + main() diff --git a/Quorum/entry_points/check_proposal_id.py b/Quorum/entry_points/check_proposal_id.py new file mode 100644 index 0000000..719562d --- /dev/null +++ b/Quorum/entry_points/check_proposal_id.py @@ -0,0 +1,135 @@ +""" +Quorum Proposal Analyzer + +This script is designed to fetch and compare smart contract source code for a specific proposal ID +associated with a designated customer on a chosen blockchain chain. The workflow includes: + +1. Parsing command-line arguments to obtain the customer identifier and proposal ID. +2. Validating the customer against supported APIs. +3. Fetching all payload addresses related to the proposal using the appropriate governance API. +4. Loading the ground truth configuration for the specified customer. +5. Cloning or updating necessary Git repositories based on the configuration. +6. Executing a series of checks on each payload's smart contract source code to ensure integrity and compliance. + +Usage: + python check_proposal_id.py --customer --proposal_id + +Example: + python check_proposal_id.py --customer Aave --proposal_id 12345 +""" + +import argparse +from typing import Tuple + +from Quorum.apis.git_api.git_manager import GitManager +from Quorum.apis.governance.aave_governance import AaveGovernanceAPI +from Quorum.checks.proposal_check import proposals_check +import Quorum.utils.config_loader as ConfigLoader + + +# Mapping of supported customers to their corresponding governance API instances. +# This allows for easy extension to support additional customers in the future. +CUSTOMER_TO_API = { + "aave": AaveGovernanceAPI() +} + + +def parse_args() -> Tuple[str, int]: + """ + Parses command-line arguments required for executing proposal analysis. + + This function utilizes Python's argparse module to define and parse the necessary command-line + arguments: + --customer: Name or identifier of the customer. + --proposal_id: ID of the proposal to analyze. + + Returns: + A tuple containing: + customer (str): Customer name or identifier. + proposal_id (int): Proposal ID. + + Raises: + argparse.ArgumentError: If required arguments are missing or invalid. + """ + parser = argparse.ArgumentParser( + description="Fetch and compare smart contract source code for a given proposal." + ) + parser.add_argument( + '--customer', + type=str, + required=True, + help="Customer name or identifier (e.g., 'Aave')." + ) + parser.add_argument( + '--proposal_id', + type=int, + required=True, + help="Proposal ID to analyze (integer value)." + ) + args = parser.parse_args() + + return args.customer, args.proposal_id + + +def main() -> None: + """ + Main execution function that orchestrates fetching, cloning/updating repositories, + and performing proposal checks based on the provided command-line arguments. + + Workflow: + 1. Parse command-line arguments to retrieve customer and proposal ID. + 2. Validate the customer against supported APIs. + 3. Fetch all payload addresses associated with the proposal ID using the governance API. + 4. Load the ground truth configuration for the specified customer. + 5. Initialize GitManager with customer and configuration, then clone or update repositories. + 6. Retrieve price feed providers from the configuration. + 7. Execute proposal checks for each set of payload addresses. + + Raises: + ValueError: If the specified customer is not supported. + FileNotFoundError: If the customer configuration file does not exist. + requests.HTTPError: If any of the HTTP requests to fetch data fail. + """ + # Parse command-line arguments + customer, proposal_id = parse_args() + + # Normalize customer identifier to lowercase for consistent mapping + customer_key = customer.lower() + + # Validate if the specified customer is supported + if customer_key not in CUSTOMER_TO_API: + raise ValueError(f"Customer '{customer}' is not supported. Supported customers: {list(CUSTOMER_TO_API.keys())}.") + + # Retrieve the appropriate governance API instance for the customer + api = CUSTOMER_TO_API[customer_key] + + # Fetch all payload addresses associated with the given proposal ID + payloads_addresses = api.get_all_payloads_addresses(proposal_id) + + # Load the ground truth configuration for the specified customer + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Iterate over each payload's data and perform proposal checks + for payload_data in payloads_addresses: + """ + payload_data is expected to be an object with at least the following attributes: + - chain: The blockchain chain associated with the payload. + - addresses: A list of smart contract addresses associated with the payload. + """ + proposals_check( + customer=customer, + chain=payload_data.chain, + proposal_addresses=payload_data.addresses, + providers=price_feed_providers + ) + + +if __name__ == "__main__": + main() diff --git a/Quorum/utils/arg_validations.py b/Quorum/utils/arg_validations.py index bfaada5..23596af 100644 --- a/Quorum/utils/arg_validations.py +++ b/Quorum/utils/arg_validations.py @@ -1,6 +1,11 @@ import re from pathlib import Path from argparse import ArgumentTypeError +from json import JSONDecodeError +import json5 as json +from typing import Any + +import Quorum.utils.pretty_printer as pp def validate_address(address: str) -> str: @@ -13,4 +18,21 @@ def validate_address(address: str) -> str: def validate_path(path: Path) -> Path: if not path.exists(): raise ArgumentTypeError(f'Could not find path at {path}.') - return path \ No newline at end of file + return path + +def load_config(config_path: str) -> dict[str, Any] | None: + """ + Load and parse the JSON configuration file. + + Args: + config_path (str): Path to the JSON configuration file. + + Returns: + dict[str, Any]: Parsed JSON data. + """ + try: + with open(config_path, 'r') as file: + config_data = json.load(file) + return config_data + except (FileNotFoundError, JSONDecodeError) as e: + pp.pretty_print(f"Failed to parse given config file {config_path}:\n{e}", pp.Colors.FAILURE) diff --git a/setup.py b/setup.py index ad12872..3328c64 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,8 @@ def read_version() -> str: entry_points={ "console_scripts": [ "CheckProposal=Quorum.entry_points.check_proposal:main", + "CheckProposalConfig=Quorum.entry_points.check_proposal_config:main", + "CheckProposalId=Quorum.entry_points.check_proposal_id:main", "IPFSValidator=Quorum.entry_points.ipfs_validator:main", "CreateReport=Quorum.entry_points.create_report:main", "SetupQuorum=Quorum.entry_points.setup_quorum:main", From bea048652b902a4841955c80dd99655c7ef6186b Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 5 Jan 2025 13:59:58 +0200 Subject: [PATCH 07/24] Make README understandable --- README.md | 382 +++++++++++++++++++++++++----------------------------- 1 file changed, 173 insertions(+), 209 deletions(-) diff --git a/README.md b/README.md index 77b0a8b..9e9dd1f 100644 --- a/README.md +++ b/README.md @@ -1,248 +1,250 @@ +Below is an updated, more succinct **Readme** for Quorum, organized as a clear step-by-step guide for new users. It consolidates the existing information and provides a logical flow to help you get started quickly and confidently. + +--- + # Quorum -Quorum is an open-source Python utility designed to verify the integrity of smart contracts deployed on blockchains. It fetches contract code directly from the blockchain and compares it with the official version provided by developers or customers in their GitHub repositories. This process helps identify discrepancies between the on-chain and official code, ensuring the contract deployed on the blockchain matches the intended version. By automating code comparison and streamlining the review of governance proposals, Quorum enhances the security and trustworthiness of smart contracts, helping users quickly detect unauthorized changes or errors. +Quorum is an open-source Python utility that ensures the integrity of smart contracts deployed on blockchains. By comparing on-chain code against known official repositories, Quorum helps detect unauthorized changes, bolstering the security and trustworthiness of decentralized systems. + +## Key Features +1. **Fetch & Compare Smart Contract Source Codes:** + - Retrieves source code directly from various block explorers via contract addresses. + - Generates unified diffs highlighting differences between local and fetched source codes. + +2. **Repository & Code Verification:** + - Compare code against audited or reviewed repositories to confirm authenticity. + - Automates repository management (clone & update) based on your configuration. + +3. **Global Variable Check:** + - Ensures all unmatched contracts’ global variables are constant or immutable. + +4. **Feed Price Check:** + - Validates that the contract feed price is listed on recognized providers like Chainlink or Chronicle. + +5. **New Listing Check:** + - Checks if a given proposal introduces a new asset listing on the protocol. + +6. **Quick Setup Command:** + - Generates essential configuration files (`.env.example`, `ground_truth.json`, `execution.json`, etc.) + - Guides you through environment variable and repository configuration steps. -## Features -- **Fetch Smart Contract Source Codes:** Retrieve source code directly from various blockchains using contract addresses. -- **Compare Local and Remote Codes:** Generate unified diffs to highlight differences between local and remote source codes. -- **Verify Code Against Known Reviewed Repositories:** Generate diffs against specifically defined trusted auditor's repositories. -- **Global Variable Check:** Ensure all global variables in unmatched contracts are either constant or immutable. -- **Feed Price Check:** Verify the feed price of a contract is mentioned on ChainLink. -- **New Listing Check:** Check if proposal contains a new Listing. -- **Automated Repository Management:** Clone or update repositories based on customer configurations. -- **Quick Setup Command:** Streamline initial configuration with a single setup command that generates necessary files and guides proper setup. +--- ## Prerequisites -- Python 3.11 or higher + +- **Python 3.11 or higher** + Quorum requires Python 3.11+, as it utilizes features from the most recent Python release. + +--- ## Installation -You can install Quorum directly from GitHub using pip: +### Via `pip` -```sh +```bash pip install git+ssh://git@github.com/Certora/Quorum.git ``` -Or clone the repository: +### Or clone the repository: -```sh +```bash git clone git@github.com:Certora/Quorum.git ``` -## Quick Setup +--- -To simplify the initial configuration, Quorum provides a setup command that generates essential configuration files and guides you through the setup process. +## Quick Setup -### Using the Setup Command +Quorum offers a convenient setup command to streamline initial configuration by creating required files and providing guidance. -Run the following command in your desired working directory (defaults to the current directory if not specified): +### 1. Run Setup Command -```sh +```bash SetupQuorum [working_directory] ``` -- **`working_directory`**: *(Optional)* Path to the desired working directory. Defaults to the current directory if not provided. +- **`working_directory`** *(Optional)*: Path where Quorum’s configuration files will be placed. If omitted, the current directory is used. -**Example:** - -```sh +**Example**: +```bash SetupQuorum ./my_quorum_project ``` -This command will: -- Copy the following template files to your working directory: - - `ground_truth.json` - - `execution.json` - - `.env.example` - - `README.md` -- Provide guidance through comments within the configuration files, and a detailed README file to help you properly configure Quorum. - -### Post-Setup Configuration - -After running the setup command, perform the following steps: +This action will: +- Create the specified (or default) directory if it doesn’t exist. +- Copy **four** template files: + 1. `ground_truth.json` + 2. `execution.json` + 3. `.env.example` + 4. `README.md` +- Provide inline comments within these files for guidance. -1. **Configure Environment Variables:** (Optional) - This step is optional if you prefer to set environment variables manually as described in the [Environment Variables](#environment-variables) section. +### 2. Post-Setup Configuration - Edit the `.env` file to include your actual API keys and desired paths: - - ```sh +1. **Environment Variables** + Edit the `.env` file (or your shell profile) with your actual API keys and custom paths: + ```bash export ETHSCAN_API_KEY="your_etherscan_api_key" export ANTHROPIC_API_KEY="your_anthropic_api_key" export QUORUM_PATH="/path/to/your/quorum_directory" ``` -2. **Fill Out Configuration Files:** +2. **Configuration Files** + - **`ground_truth.json`**: Define repositories and providers (e.g., price feed providers, token validation). + - **`execution.json`**: Specify proposal addresses to be checked for different chains. + - **`README.md`**: An auto-generated resource explaining your next steps. - - **`ground_truth.json`**: Define repositories and providers for each protocol. - - **`execution.json`**: Specify proposal addresses for each network. - - **`README.md`**: Follow the included guide to understand installation, configuration, available flags, and the checks performed by Quorum. +--- ## Clarifications -As part of the tool's process, Quorum uses `solcx` to parse contract code to AST. The version of `solcx` used is the latest. If the contract code is not compatible with the latest version of `solcx`, the tool will not be able to parse the contract code and will not be able to proceed with the global variable and new listing checks. +Quorum leverages `solcx` (latest version) to parse contract code into an AST. Contracts incompatible with the latest `solc` version may break checks involving AST parsing (e.g., global variable checks, new listing checks). + +--- ## Environment Variables -Quorum requires specific environment variables to function correctly. These variables can be set in your shell or defined in a `.env` file. +To fully enable Quorum’s checks, set the following: + +### Required Variables +- **`ETHSCAN_API_KEY`**: Your Etherscan API key (for block explorer queries). +- **`ANTHROPIC_API_KEY`**: Required if you intend to use advanced LLM-based checks (e.g., new listing first deposit checks). +- **`QUORUM_PATH`**: Directory path where Quorum stores cloned repos, diffs, logs, etc. + +### Setting Variables -### Required Environment Variables +1. **Shell Environment:** + + ```bash + export ETHSCAN_API_KEY="your_etherscan_api_key" + export ANTHROPIC_API_KEY="your_anthropic_api_key" + export QUORUM_PATH="/path/to/quorum_artifacts" + ``` -- **ETHSCAN_API_KEY:** API key for Etherscan. -- **ANTHROPIC_API_KEY:** API key for Anthropic (required for advanced new listing first deposit checks). -- **QUORUM_PATH:** Path to specify where the repositories and diffs will be saved. +2. **`.env` File:** + + ``` + ETHSCAN_API_KEY=your_etherscan_api_key + ANTHROPIC_API_KEY=your_anthropic_api_key + QUORUM_PATH="/path/to/quorum_artifacts" + ``` + +*(This file is automatically created by `SetupQuorum` if not already present.)* + +--- + +## Usage -### Setting Environment Variables +Quorum provides multiple entry points to handle different aspects of proposal checks and code validation. Below is an overview of each command, with examples. -**Using Shell:** +### 1. **CheckProposal** -```sh -export ETHSCAN_API_KEY="your_etherscan_api_key" -export ANTHROPIC_API_KEY="your_anthropic_api_key" -export QUORUM_PATH="/path/to/artifacts" +**Purpose:** Analyzes a single proposal by address for a specific customer on a given chain. + +```bash +CheckProposal --customer "Aave" --chain "Ethereum" --proposal_address "0xAD6..." ``` -**Using `.env` File:** +### 2. **CheckProposalConfig** + +**Purpose:** Processes multiple proposals in bulk using a JSON config file. + +```bash +CheckProposalConfig --config "/path/to/config.json" +``` +*(See “**Example Usage with Config File**” for a sample config.)* -After running the setup command, a `.env` file will be present. fill in the required values: +### 3. **CheckProposalId** -Then edit `.env`: +**Purpose:** Looks up all payload addresses for a single proposal ID (useful for proposals containing multiple payloads). -```sh -ETHSCAN_API_KEY=your_etherscan_api_key -ANTHROPIC_API_KEY=your_anthropic_api_key -QUORUM_PATH="/path/to/artifacts" +```bash +CheckProposalId --customer "Aave" --proposal_id 137 ``` -## Usage +### 4. **IPFSValidator** -To run the tool, use the command line: +**Purpose:** Validates whether the IPFS description content aligns with the actual on-chain payload. Uses LLM-based analysis. -```sh -CheckProposal --customer "CustomerName" --chain "ChainName" --proposal_address "Address" +```bash +IPFSValidator --proposal_id 132 --chain "Ethereum" --proposal_address "0xAD6..." ``` -OR +### 5. **CreateReport** + +**Purpose:** Generates a human-readable report of the proposal details, leveraging Jinja2 templates. -```sh -python3 Quorum/check_proposal.py --customer "CustomerName" --chain "ChainName" --proposal_address "Address" +```bash +CreateReport --proposal_id 137 \ + --template "Quorum/auto_report/AaveReportTemplate.md.j2" \ + --generate_report_path "reports/v3-137.md" ``` -Replace `CustomerName` with the customer identifier, `ChainName` with the blockchain chain, and `Address` with the proposal address. +### 6. **SetupQuorum** -### Example Usage with Config File +**Purpose:** Bootstraps your Quorum environment, creating `.env`, `ground_truth.json`, `execution.json`, and an initial `README.md`. + +```bash +SetupQuorum --working_dir "/home/user/quorum_project" +``` -You can also execute multiple tasks using a configuration file: +*(Refer to “**Quick Setup**” for details.)* -**Example config file `config.json`:** +--- + +## Example Usage with Config File + +For bulk execution, create a config file (e.g., `config.json`) with the following format: ```json { "Aave": { - "Ethereum": { - "Proposals": [ - "0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637" - ] - }, - "Arbitrum": { - "Proposals": [ - "0x22ca2Dd3063189F9E7e76fA3078E2d916B3998b7" - ] - }, - "Avalanche": { - "Proposals": [] - }, - "Base": { - "Proposals": [ - "0x6B96B41a531713a141F6EcBbae80715601d0e456" - ] - }, - "BNBChain": { - "Proposals": [ - "0xb4F2786984093eaE1D6Be2B4F8c8e3c2cb018b54" - ] - }, - "Gnosis": { - "Proposals": [] - }, - "Metis": { - "Proposals": [] - }, - "Optimism": { - "Proposals": [] - }, - "Polygon": { - "Proposals": [ - "0x2dbBe7E30CD959A192FeFCEd9A5ae681d540deB4" - ] - }, - "Scroll": { - "Proposals": [ - "0x9d9892256dF8f97d0c15F4494aa5D44D376CC749" - ] - }, - "zkSync": { - "Proposals": [] - } + "Ethereum": { "Proposals": [ "0xAD6..." ] }, + "Arbitrum": { "Proposals": [ "0x22ca2..." ] }, + ... } } ``` -**To run using the config file:** +Then run: -```sh -python3 Quorum/check_proposal.py --config path/to/config.json +```bash +CheckProposalConfig --config config.json ``` -Or if you used the pip installation: - -```sh -CheckProposal --config path/to/config.json -``` +*(Chains without proposals are automatically skipped.)* -**Note:** If the "Proposals" list for a particular chain is empty, the task for that chain will be skipped. This allows you to include or exclude chains from processing without modifying the code. +--- -## Configuration +## Configuration Details -The `ground_truth.json` file defines the repositories for each customer. It should be located under the `QUORUM_PATH`. If not found, a default `ground_truth.json` configuration will be created. +### ground_truth.json -### Template for `ground_truth.json`: +Defines each protocol’s repositories and providers: ```json { "ProtocolName": { "dev_repos": [ - "https://github.com/organization/repository1", - "https://github.com/organization/repository2" + "https://github.com/org/repo1", + "https://github.com/org/repo2" ], - "review_repo": "https://github.com/organization/review-repository", + "review_repo": "https://github.com/org/review", "price_feed_providers": ["Chainlink", "Chronicle"], "token_validation_providers": ["Coingecko"] } } ``` -**Fields explanation:** -- `ProtocolName`: Your protocol or organization name -- `dev_repos`: List of GitHub repositories containing your protocol's source code -- `review_repo`: Repository containing pre-deployment code for review -- `price_feed_providers`: List of supported price feed providers (Chainlink, Chronicle) -- `token_validation_providers`: List of supported token validation providers (Coingecko) +### Currently Supported Providers +- **Price Feeds**: Chainlink, Chronicle +- **Token Validation**: Coingecko -### Current Supported Providers - -**Price Feed Providers:** -- Chainlink -- Chronicle - -**Token Validation Providers:** -- Coingecko +--- ## Artifacts Structure -Quorum generates and organizes artifacts in a structured manner under the `QUORUM_PATH` directory. Here is a general overview of the structure: - -### Directory Structure +All artifacts (cloned repos, diffs, logs) are stored under `QUORUM_PATH`. Below is a typical folder hierarchy: ``` QUORUM_PATH/ @@ -251,76 +253,38 @@ QUORUM_PATH/ │ ├── modules/ │ │ ├── repository1/ │ │ ├── repository2/ -│ │ ├── ... │ ├── checks/ │ │ ├── ChainName/ -│ │ │ ├── ProposalAddress1/ -│ │ │ │ ├── DiffCheck_datetime/ -│ │ │ │ │ ├── file1.patch -│ │ │ │ │ ├── file2.patch -│ │ │ │ ├── FeedPriceCheck_datetime/ -│ │ │ │ │ ├── file1.json -│ │ │ │ ├── GlobalVariableCheck_datetime/ -│ │ │ │ │ ├── file1.json -│ │ │ │ │ ├── ... -│ │ │ │ ├── NewListingCheck_datetime/ -│ │ │ │ │ ├── file1.json -│ │ │ │ ├── ... -│ │ │ ├── ProposalAddress2/ -│ │ │ ├── ... -│ │ ├── ... -│ │ ├── ProposalAddressN/ +│ │ │ ├── ProposalAddress/ +│ │ │ │ ├── DiffCheck_/ +│ │ │ │ ├── FeedPriceCheck_/ +│ │ │ │ ├── GlobalVariableCheck_/ +│ │ │ │ ├── NewListingCheck_/ │ │ │ ├── ... +│ ├── execution.json +│ └── ground_truth.json ``` -### Description - -- **CustomerName/**: This directory is named after the customer, representing the context or organization for which the analysis is performed. Each customer has its own directory. - - - **checks/**: Contains the diffs and global variable checks generated for each smart contract address analyzed. Each subdirectory is named after the contract's address and contains patch files highlighting differences between local and remote source codes, as well as JSON files documenting any global variables that are not constant or immutable. - - - **modules/**: This directory stores the cloned repositories for the customer. Each subdirectory corresponds to a specific repository associated with the customer, containing the source code and related files. - - - **execution.json**: This file stores the configuration and results of the last execution, including details like which proposals were checked and any findings or issues encountered. - - - **ground_truth.json**: A configuration file specifying the repositories to be managed for the customer. This file can be customized to include the URLs of the repositories related to the customer. - -### Example +1. **`CustomerName/`**: Each customer has a dedicated folder. +2. **`modules/`**: Contains cloned Git repositories. +3. **`checks/`**: Contains patch files (diffs) and JSON logs from the checks performed. +4. **`execution.json`**: Tracks the proposals processed in the last run. +5. **`ground_truth.json`**: Core configuration defining the official repositories and providers. -For instance, the structure under the `QUORUM_PATH/Aave/` directory might look like: - -``` -Aave/ -├── checks/ -│ ├── 0x065DF1F9d0aeDEa11E6d059ce29e91d2Abed59fA/ -│ │ ├── diffs_20240801_105150/ -│ │ │ ├── AaveV3Ethereum.patch -│ │ ├── global_check_20240801_105150/ -│ │ │ ├── AaveV3Ethereum.json -│ ├── 0x564Dfd09eBB63F7e468401AffE2d8c2cDD08D68D/ -│ │ ├── ... -│ ├── 0x683FdF51d5898F92317F870B25a6A4dF67dC58Ab/ -│ │ ├── ... -│ ├── 0xF0221Fc5a2F825bbF6F994f30743aD5AAC66cd4E/ -│ │ ├── ... -├── modules/ -│ ├── aave-address-book/ -│ ├── aave-helpers/ -│ ├── aave-v3-origin/ -├── execution.json -├── ground_truth.json -``` - -In this example, each proposal address under the `checks/` directory contains diff files that highlight the differences between the local and fetched source codes, as well as global variable check results. The `modules/` directory contains the repositories relevant to the customer "Aave," and the `execution.json` and `ground_truth.json` files hold metadata and configuration details. +--- ## License -Quorum is released under the [MIT License](LICENSE). +Quorum is licensed under the [MIT License](LICENSE). ## Contributing -Contributions are welcome! Please open an issue or submit a pull request on GitHub. +Contributions are welcome! Please open an issue or submit a pull request on [GitHub](https://github.com/Certora/Quorum). ## Acknowledgments -- Thanks to all contributors and the open-source community. +- Special thanks to all contributors and the open-source community for their support. + +--- + +**Happy Auditing!** If you have any questions or run into issues, please don’t hesitate to create a GitHub issue or open a discussion. \ No newline at end of file From 51203f3279f14be16feda1cf71f62005a33d3d57 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 5 Jan 2025 14:08:38 +0200 Subject: [PATCH 08/24] Address comments --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 9e9dd1f..e72c533 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ -Below is an updated, more succinct **Readme** for Quorum, organized as a clear step-by-step guide for new users. It consolidates the existing information and provides a logical flow to help you get started quickly and confidently. - ---- - # Quorum Quorum is an open-source Python utility that ensures the integrity of smart contracts deployed on blockchains. By comparing on-chain code against known official repositories, Quorum helps detect unauthorized changes, bolstering the security and trustworthiness of decentralized systems. From 739e281f9f3b29d7aa5f386f80f9a3ea08fd637f Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 12:44:30 +0200 Subject: [PATCH 09/24] Intoduce command pattern --- Quorum/entry_points/check_proposal.py | 121 ---------------- Quorum/entry_points/check_proposal_config.py | 111 -------------- Quorum/entry_points/check_proposal_id.py | 135 ------------------ Quorum/entry_points/cli_arguments.py | 91 ++++++++++++ Quorum/entry_points/create_report.py | 54 ------- .../entry_points/implementations/__init__.py | 0 .../implementations/check_proposal.py | 56 ++++++++ .../implementations/check_proposal_config.py | 71 +++++++++ .../implementations/check_proposal_id.py | 75 ++++++++++ .../implementations/create_report.py | 49 +++++++ .../{ => implementations}/ipfs_validator.py | 54 ++++--- .../{ => implementations}/setup_quorum.py | 54 +++---- Quorum/entry_points/quorum_cli.py | 107 ++++++++++++++ Quorum/tests/test_auto_report.py | 2 +- setup.py | 7 +- version | 2 +- 16 files changed, 498 insertions(+), 491 deletions(-) delete mode 100644 Quorum/entry_points/check_proposal.py delete mode 100644 Quorum/entry_points/check_proposal_config.py delete mode 100644 Quorum/entry_points/check_proposal_id.py create mode 100644 Quorum/entry_points/cli_arguments.py delete mode 100644 Quorum/entry_points/create_report.py create mode 100644 Quorum/entry_points/implementations/__init__.py create mode 100644 Quorum/entry_points/implementations/check_proposal.py create mode 100644 Quorum/entry_points/implementations/check_proposal_config.py create mode 100644 Quorum/entry_points/implementations/check_proposal_id.py create mode 100644 Quorum/entry_points/implementations/create_report.py rename Quorum/entry_points/{ => implementations}/ipfs_validator.py (67%) rename Quorum/entry_points/{ => implementations}/setup_quorum.py (55%) create mode 100644 Quorum/entry_points/quorum_cli.py diff --git a/Quorum/entry_points/check_proposal.py b/Quorum/entry_points/check_proposal.py deleted file mode 100644 index e450762..0000000 --- a/Quorum/entry_points/check_proposal.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Quorum Proposal Checker - -This script is designed to fetch and compare smart contract source code for a specific proposal address -on a chosen blockchain chain for a designated customer. The workflow includes: - -1. Parsing command-line arguments to obtain the customer identifier, blockchain chain, and proposal address. -2. Loading the ground truth configuration for the specified customer. -3. Cloning or updating necessary Git repositories based on the configuration. -4. Executing a series of checks on the proposal's smart contract source code to ensure integrity and compliance. - -Usage: - python check_proposal.py --customer --chain --proposal_address - -Example: - python check_proposal.py --customer Aave --chain Ethereum --proposal_address 0x1234567890abcdef1234567890abcdef12345678 -""" - -import argparse -from typing import Tuple - -from Quorum.utils.chain_enum import Chain -from Quorum.apis.git_api.git_manager import GitManager -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader -import Quorum.utils.arg_validations as arg_valid - - -def parse_args() -> Tuple[str, Chain, str]: - """ - Parses command-line arguments required for executing proposal checks. - - This function utilizes Python's argparse module to define and parse the necessary command-line - arguments: - --customer: Name or identifier of the customer. - --chain: Blockchain chain to target, must be one of the defined Chain enum values. - --proposal_address: Ethereum proposal address, validated using `arg_valid.validate_address`. - - Returns: - A tuple containing: - customer (str): Customer name or identifier. - chain (Chain): Selected blockchain chain. - proposal_address (str): Validated Ethereum proposal address. - - Raises: - argparse.ArgumentError: If required arguments are missing or invalid. - """ - parser = argparse.ArgumentParser( - description="Fetch and compare smart contract source code for a given proposal." - ) - parser.add_argument( - '--customer', - type=str, - required=True, - help="Customer name or identifier." - ) - parser.add_argument( - '--chain', - type=Chain, - choices=list(Chain), - required=True, - help="Blockchain chain to target." - ) - parser.add_argument( - '--proposal_address', - type=arg_valid.validate_address, - required=True, - help="Ethereum proposal address (e.g., 0x...)." - ) - args = parser.parse_args() - - return args.customer, args.chain, args.proposal_address - - -def main() -> None: - """ - Main execution function that orchestrates fetching, cloning/updating repositories, - and performing proposal checks based on the provided command-line arguments. - - Workflow: - 1. Parse command-line arguments to retrieve customer, chain, and proposal address. - 2. Load the ground truth configuration for the specified customer. - 3. Initialize and update Git repositories as per the configuration. - 4. Execute the proposal checks for the provided proposal address using the loaded configuration. - - Raises: - ValueError: If any of the required arguments (customer, chain, proposal_address) are missing. - FileNotFoundError: If the customer configuration file does not exist. - requests.HTTPError: If any of the HTTP requests to fetch data fail. - """ - # Parse command-line arguments - customer, chain, proposal_address = parse_args() - - # Ensure all required arguments are provided - if not (customer and chain and proposal_address): - raise ValueError( - "Customer, chain, and proposal_address must be specified." - " Provide all three arguments when not using a config file." - ) - - # Load the customer's ground truth configuration - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Execute proposal checks - proposals_check( - customer=customer, - chain=chain, - proposal_addresses=[proposal_address], - providers=price_feed_providers - ) - - -if __name__ == "__main__": - main() diff --git a/Quorum/entry_points/check_proposal_config.py b/Quorum/entry_points/check_proposal_config.py deleted file mode 100644 index 6e31d75..0000000 --- a/Quorum/entry_points/check_proposal_config.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Quorum JSON Configuration Proposal Analyzer - -This script is designed to fetch and compare smart contract source code based on a JSON configuration file -for various customers and their respective blockchain chains. The workflow includes: - -1. Parsing command-line arguments to obtain the path to a JSON configuration file. -2. Loading the ground truth configuration for each specified customer. -3. Cloning or updating necessary Git repositories based on the configuration. -4. Executing a series of proposal checks on the provided proposal addresses to ensure integrity and compliance. - -Usage: - python check_proposal_config.py --config - -Example: - python check_proposal_config.py --config execution.json -""" - -import argparse -from typing import Dict, Any - -from Quorum.apis.git_api.git_manager import GitManager -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader -import Quorum.utils.arg_validations as arg_valid -from Quorum.utils.chain_enum import Chain - - -def parse_args() -> Dict[str, Any]: - """ - Parses command-line arguments required for executing proposal analysis based on a JSON configuration file. - - This function utilizes Python's argparse module to define and parse the necessary command-line - arguments: - --config: Path to the JSON configuration file containing customer and proposal details. - - Returns: - dict: Parsed JSON data from the configuration file. - - Raises: - argparse.ArgumentError: If the provided configuration file path is invalid or the file cannot be loaded. - """ - parser = argparse.ArgumentParser( - description="Fetch and compare smart contract source code based on a JSON configuration file." - ) - parser.add_argument( - '--config', - type=arg_valid.load_config, - required=True, - help="Path to the JSON configuration file." - ) - args = parser.parse_args() - - return args.config - - -def main() -> None: - """ - Main execution function that orchestrates fetching, cloning/updating repositories, - and performing proposal checks based on the provided JSON configuration file. - - Workflow: - 1. Parse command-line arguments to retrieve the configuration data. - 2. Iterate over each customer and their associated chain and proposal information. - 3. Load the ground truth configuration for each customer. - 4. Initialize GitManager with customer and configuration, then clone or update repositories. - 5. Retrieve price feed providers from the configuration. - 6. Execute proposal checks for each set of proposal addresses on the specified chains. - - Raises: - ValueError: If the specified customer is not supported. - FileNotFoundError: If the customer configuration file does not exist. - requests.HTTPError: If any of the HTTP requests to fetch data fail. - """ - # Parse command-line arguments to get the configuration data - config_data: Dict[str, Any] = parse_args() - - # Iterate over each customer and their respective chain and proposal information in the config - for customer, chain_info in config_data.items(): - # Load the ground truth configuration for the specified customer - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Iterate over each blockchain chain and its associated proposals for the customer - for chain, proposals in chain_info.items(): - try: - # Validate and convert the chain identifier to the Chain enum - chain_enum = Chain(chain) - except ValueError as e: - # Handle unsupported or invalid chain identifiers - raise ValueError(f"Unsupported or invalid chain '{chain}' for customer '{customer}': {e}") - - # Check if there are any proposals to process for the current chain - if proposals.get("Proposals"): - # Execute proposal checks for the specified customer, chain, and proposal addresses - proposals_check( - customer=customer, - chain=chain_enum, - proposal_addresses=proposals["Proposals"], - providers=price_feed_providers - ) - - -if __name__ == "__main__": - main() diff --git a/Quorum/entry_points/check_proposal_id.py b/Quorum/entry_points/check_proposal_id.py deleted file mode 100644 index 719562d..0000000 --- a/Quorum/entry_points/check_proposal_id.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Quorum Proposal Analyzer - -This script is designed to fetch and compare smart contract source code for a specific proposal ID -associated with a designated customer on a chosen blockchain chain. The workflow includes: - -1. Parsing command-line arguments to obtain the customer identifier and proposal ID. -2. Validating the customer against supported APIs. -3. Fetching all payload addresses related to the proposal using the appropriate governance API. -4. Loading the ground truth configuration for the specified customer. -5. Cloning or updating necessary Git repositories based on the configuration. -6. Executing a series of checks on each payload's smart contract source code to ensure integrity and compliance. - -Usage: - python check_proposal_id.py --customer --proposal_id - -Example: - python check_proposal_id.py --customer Aave --proposal_id 12345 -""" - -import argparse -from typing import Tuple - -from Quorum.apis.git_api.git_manager import GitManager -from Quorum.apis.governance.aave_governance import AaveGovernanceAPI -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader - - -# Mapping of supported customers to their corresponding governance API instances. -# This allows for easy extension to support additional customers in the future. -CUSTOMER_TO_API = { - "aave": AaveGovernanceAPI() -} - - -def parse_args() -> Tuple[str, int]: - """ - Parses command-line arguments required for executing proposal analysis. - - This function utilizes Python's argparse module to define and parse the necessary command-line - arguments: - --customer: Name or identifier of the customer. - --proposal_id: ID of the proposal to analyze. - - Returns: - A tuple containing: - customer (str): Customer name or identifier. - proposal_id (int): Proposal ID. - - Raises: - argparse.ArgumentError: If required arguments are missing or invalid. - """ - parser = argparse.ArgumentParser( - description="Fetch and compare smart contract source code for a given proposal." - ) - parser.add_argument( - '--customer', - type=str, - required=True, - help="Customer name or identifier (e.g., 'Aave')." - ) - parser.add_argument( - '--proposal_id', - type=int, - required=True, - help="Proposal ID to analyze (integer value)." - ) - args = parser.parse_args() - - return args.customer, args.proposal_id - - -def main() -> None: - """ - Main execution function that orchestrates fetching, cloning/updating repositories, - and performing proposal checks based on the provided command-line arguments. - - Workflow: - 1. Parse command-line arguments to retrieve customer and proposal ID. - 2. Validate the customer against supported APIs. - 3. Fetch all payload addresses associated with the proposal ID using the governance API. - 4. Load the ground truth configuration for the specified customer. - 5. Initialize GitManager with customer and configuration, then clone or update repositories. - 6. Retrieve price feed providers from the configuration. - 7. Execute proposal checks for each set of payload addresses. - - Raises: - ValueError: If the specified customer is not supported. - FileNotFoundError: If the customer configuration file does not exist. - requests.HTTPError: If any of the HTTP requests to fetch data fail. - """ - # Parse command-line arguments - customer, proposal_id = parse_args() - - # Normalize customer identifier to lowercase for consistent mapping - customer_key = customer.lower() - - # Validate if the specified customer is supported - if customer_key not in CUSTOMER_TO_API: - raise ValueError(f"Customer '{customer}' is not supported. Supported customers: {list(CUSTOMER_TO_API.keys())}.") - - # Retrieve the appropriate governance API instance for the customer - api = CUSTOMER_TO_API[customer_key] - - # Fetch all payload addresses associated with the given proposal ID - payloads_addresses = api.get_all_payloads_addresses(proposal_id) - - # Load the ground truth configuration for the specified customer - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Iterate over each payload's data and perform proposal checks - for payload_data in payloads_addresses: - """ - payload_data is expected to be an object with at least the following attributes: - - chain: The blockchain chain associated with the payload. - - addresses: A list of smart contract addresses associated with the payload. - """ - proposals_check( - customer=customer, - chain=payload_data.chain, - proposal_addresses=payload_data.addresses, - providers=price_feed_providers - ) - - -if __name__ == "__main__": - main() diff --git a/Quorum/entry_points/cli_arguments.py b/Quorum/entry_points/cli_arguments.py new file mode 100644 index 0000000..540c3d1 --- /dev/null +++ b/Quorum/entry_points/cli_arguments.py @@ -0,0 +1,91 @@ +from pydantic import BaseModel +from typing import Any +from pathlib import Path + +from Quorum.utils.chain_enum import Chain +import Quorum.utils.arg_validations as arg_valid + + +class Argument(BaseModel): + name: str + type: Any + required: bool + help: str + default: Any | None = None + nargs: str | None = None + + +CUSTOMER_ARGUMENT = Argument( + name='--customer', + type=str, + required=True, + help="Customer name or identifier." +) + + +CHAIN_ARGUMENT = Argument( + name='--chain', + type=Chain, + required=True, + help="Blockchain chain to target." +) + + +PROPOSAL_ADDRESS_ARGUMENT = Argument( + name='--proposal_address', + type=arg_valid.validate_address, + required=True, + help="Ethereum proposal address." +) + + +PROPOSAL_ID_ARGUMENT = Argument( + name='--proposal_id', + type=int, + required=True, + help="ID of the proposal." +) + + +CONFIG_ARGUMENT = Argument( + name='--config', + type=arg_valid.load_config, + required=True, + help="Path to the JSON configuration file." +) + + +TEMPLATE_ARGUMENT = Argument( + name='--template', + type=Path, + required=False, + help="Path to the Jinja2 template file.", + default=Path(__file__).parent.parent / 'auto_report/AaveReportTemplate.md.j2' +) + + +GENERATE_REPORT_PATH_ARGUMENT = Argument( + name='--generate_report_path', + type=Path, + required=False, + help="Path to save the generated report." +) + + +PROMPT_TEMPLATES_ARGUMENT = Argument( + name='--prompt_templates', + type=str, + required=False, + help="Jinja templates for prompting the LLM.", + default=['ipfs_validation_prompt_part1.j2', "ipfs_validation_prompt_part2.j2"], + nargs="+" +) + + +WORKING_DIR_ARGUMENT = Argument( + name='--working_dir', + type=Path, + required=False, + help="Where to create the Quorum project.", + default=Path.cwd() / 'quorum_project' +) diff --git a/Quorum/entry_points/create_report.py b/Quorum/entry_points/create_report.py deleted file mode 100644 index 538a325..0000000 --- a/Quorum/entry_points/create_report.py +++ /dev/null @@ -1,54 +0,0 @@ -import Quorum.auto_report.aave_tags as aave_tags -import Quorum.utils.pretty_printer as pprinter - -import argparse -from pathlib import Path - -from jinja2 import Environment, FileSystemLoader - - -DEFAULT_TEMPLATE_PATH = Path(__file__).parent.parent / 'auto_report/AaveReportTemplate.md.j2' - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description='This tool generates automatic proposal reports.') - parser.add_argument('--proposal_id', required=True, type=int, help='The proposal id to generate report to.') - parser.add_argument('--template', default=DEFAULT_TEMPLATE_PATH, help='The report template to use.') - parser.add_argument('--generate_report_path', type=Path, help='Specify where to save the report.') - - args = parser.parse_args() - - if not Path(args.template).exists(): - raise FileNotFoundError(f'could not find template at {args.template}.') - - if args.generate_report_path is None: - args.generate_report_path = Path(f'v3-{args.proposal_id}.md') - - return args - - -def create_report(proposal_id: int, template: Path, generate_report_path: Path): - pprinter.pretty_print(f'Generating a report using template in {template}', pprinter.Colors.INFO) - env = Environment(loader=FileSystemLoader(template.parent)) - env.globals.update(zip=zip) - template = env.get_template(template.name) - - pprinter.pretty_print(f'Retrieving tag information for proposal {proposal_id}', pprinter.Colors.INFO) - tags = aave_tags.get_aave_tags(proposal_id) - pprinter.pretty_print(f'Tag information retrieved', pprinter.Colors.INFO) - - report = template.render(tags) - - with open(generate_report_path, 'w') as f: - f.write(report) - - pprinter.pretty_print(f'Created report at {generate_report_path}.', pprinter.Colors.SUCCESS) - - -def main(): - args = parse_args() - create_report(args.proposal_id, args.template, args.generate_report_path) - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/Quorum/entry_points/implementations/__init__.py b/Quorum/entry_points/implementations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Quorum/entry_points/implementations/check_proposal.py b/Quorum/entry_points/implementations/check_proposal.py new file mode 100644 index 0000000..c8a8a39 --- /dev/null +++ b/Quorum/entry_points/implementations/check_proposal.py @@ -0,0 +1,56 @@ +import argparse + +from Quorum.apis.git_api.git_manager import GitManager +from Quorum.checks.proposal_check import proposals_check +import Quorum.utils.config_loader as ConfigLoader + +def run_single(args: argparse.Namespace) -> None: + """ + Execute proposal verification for a single proposal address. + + This function processes a single proposal check by initializing necessary configurations + and git repositories, then performs validation checks on the specified proposal. + + Args: + args (argparse.Namespace): Command line arguments containing: + - customer (str): Customer identifier + - chain (str): Blockchain network identifier + - proposal_address (str): Address of the proposal to check + + Raises: + ValueError: If any of the required arguments (customer, chain, proposal_address) are missing + + Example: + args = argparse.Namespace( + customer='example_customer', + chain='ethereum', + proposal_address='0x1234...' + run_single(args) + """ + # Parse command-line arguments + customer, chain, proposal_address = args.customer, args.chain, args.proposal_address + + # Ensure all required arguments are provided + if not (customer and chain and proposal_address): + raise ValueError( + "Customer, chain, and proposal_address must be specified." + " Provide all three arguments when not using a config file." + ) + + # Load the customer's ground truth configuration + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Execute proposal checks + proposals_check( + customer=customer, + chain=chain, + proposal_addresses=[proposal_address], + providers=price_feed_providers + ) diff --git a/Quorum/entry_points/implementations/check_proposal_config.py b/Quorum/entry_points/implementations/check_proposal_config.py new file mode 100644 index 0000000..cf1b47a --- /dev/null +++ b/Quorum/entry_points/implementations/check_proposal_config.py @@ -0,0 +1,71 @@ +import argparse +from typing import Dict, Any + +from Quorum.apis.git_api.git_manager import GitManager +from Quorum.checks.proposal_check import proposals_check +import Quorum.utils.config_loader as ConfigLoader +from Quorum.utils.chain_enum import Chain + + +def run_config(args: argparse.Namespace) -> None: + """ + Processes and validates proposal configurations for different customers across blockchain networks. + + This function takes command-line arguments containing configuration data and performs the following: + - Loads ground truth configurations for each customer + - Manages Git repositories through GitManager + - Validates blockchain chains + - Executes proposal checks for specified addresses + + Args: + args (argparse.Namespace): Command-line arguments containing the configuration data + in args.config as a dictionary + + Returns: + None + + Raises: + ValueError: If an unsupported or invalid blockchain chain is specified + + Example config_data structure: + { + "customer_name": { + "chain_name": { + "Proposals": ["0x123...", "0x456..."] + } + } + } + """ + # Parse command-line arguments to get the configuration data + config_data: Dict[str, Any] = args.config + + # Iterate over each customer and their respective chain and proposal information in the config + for customer, chain_info in config_data.items(): + # Load the ground truth configuration for the specified customer + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Iterate over each blockchain chain and its associated proposals for the customer + for chain, proposals in chain_info.items(): + try: + # Validate and convert the chain identifier to the Chain enum + chain_enum = Chain(chain) + except ValueError as e: + # Handle unsupported or invalid chain identifiers + raise ValueError(f"Unsupported or invalid chain '{chain}' for customer '{customer}': {e}") + + # Check if there are any proposals to process for the current chain + if proposals.get("Proposals"): + # Execute proposal checks for the specified customer, chain, and proposal addresses + proposals_check( + customer=customer, + chain=chain_enum, + proposal_addresses=proposals["Proposals"], + providers=price_feed_providers + ) diff --git a/Quorum/entry_points/implementations/check_proposal_id.py b/Quorum/entry_points/implementations/check_proposal_id.py new file mode 100644 index 0000000..6f00981 --- /dev/null +++ b/Quorum/entry_points/implementations/check_proposal_id.py @@ -0,0 +1,75 @@ +import argparse + +from Quorum.apis.git_api.git_manager import GitManager +from Quorum.apis.governance.aave_governance import AaveGovernanceAPI +from Quorum.checks.proposal_check import proposals_check +import Quorum.utils.config_loader as ConfigLoader + + +CUSTOMER_TO_API = { + "aave": AaveGovernanceAPI() +} + + +def run_proposal_id(args: argparse.Namespace) -> None: + """ + Executes a validation process for a given proposal ID associated with a specific customer. + + This function performs several key operations: + 1. Validates the customer and retrieves their governance API + 2. Fetches payload addresses for the given proposal ID + 3. Loads customer configuration and sets up git repositories + 4. Performs proposal checks for each payload + + Args: + args (argparse.Namespace): Command line arguments containing: + - customer (str): The identifier of the customer + - proposal_id (str/int): The ID of the proposal to check + + Raises: + ValueError: If the specified customer is not supported + + Example: + args = argparse.Namespace(customer='example', proposal_id='123') + run_proposal_id(args) + """ + + # Parse command-line arguments + customer, proposal_id = args.customer, args.proposal_id + + # Normalize customer identifier to lowercase for consistent mapping + customer_key = customer.lower() + + # Validate if the specified customer is supported + if customer_key not in CUSTOMER_TO_API: + raise ValueError(f"Customer '{customer}' is not supported. Supported customers: {list(CUSTOMER_TO_API.keys())}.") + + # Retrieve the appropriate governance API instance for the customer + api = CUSTOMER_TO_API[customer_key] + + # Fetch all payload addresses associated with the given proposal ID + payloads_addresses = api.get_all_payloads_addresses(proposal_id) + + # Load the ground truth configuration for the specified customer + ground_truth_config = ConfigLoader.load_customer_config(customer) + + # Initialize GitManager with customer and configuration, then clone or update repositories + git_manager = GitManager(customer, ground_truth_config) + git_manager.clone_or_update() + + # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + # Iterate over each payload's data and perform proposal checks + for payload_data in payloads_addresses: + """ + payload_data is expected to be an object with at least the following attributes: + - chain: The blockchain chain associated with the payload. + - addresses: A list of smart contract addresses associated with the payload. + """ + proposals_check( + customer=customer, + chain=payload_data.chain, + proposal_addresses=payload_data.addresses, + providers=price_feed_providers + ) diff --git a/Quorum/entry_points/implementations/create_report.py b/Quorum/entry_points/implementations/create_report.py new file mode 100644 index 0000000..d116a72 --- /dev/null +++ b/Quorum/entry_points/implementations/create_report.py @@ -0,0 +1,49 @@ +import argparse +from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +import Quorum.auto_report.aave_tags as aave_tags +import Quorum.utils.pretty_printer as pprinter + + +def run_create_report(args: argparse.Namespace): + """ + Creates a report by applying provided tags to a template. + Args: + args (argparse.Namespace): Command line arguments containing: + - template (Path): Path to the template file + - proposal_id (int): ID of the proposal to generate report for + - generate_report_path (Path, optional): Output path for the generated report. + Defaults to 'v3-{proposal_id}.md' + Raises: + FileNotFoundError: If the template file does not exist at the specified path + The function: + 1. Validates template existence + 2. Sets default report path if none provided + 3. Loads and renders template with proposal tags + 4. Writes rendered report to specified output path + Returns: + None + """ + if not args.template.exists(): + raise FileNotFoundError(f'could not find template at {args.template}.') + + if args.generate_report_path is None: + args.generate_report_path = Path(f'v3-{args.proposal_id}.md') + + + pprinter.pretty_print(f'Generating a report using template in {args.template}', pprinter.Colors.INFO) + env = Environment(loader=FileSystemLoader(args.template.parent)) + env.globals.update(zip=zip) + template = env.get_template(args.template.name) + + pprinter.pretty_print(f'Retrieving tag information for proposal {args.proposal_id}', pprinter.Colors.INFO) + tags = aave_tags.get_aave_tags(args.proposal_id) + pprinter.pretty_print(f'Tag information retrieved', pprinter.Colors.INFO) + + report = template.render(tags) + + with open(args.generate_report_path, 'w') as f: + f.write(report) + + pprinter.pretty_print(f'Created report at {args.generate_report_path}.', pprinter.Colors.SUCCESS) diff --git a/Quorum/entry_points/ipfs_validator.py b/Quorum/entry_points/implementations/ipfs_validator.py similarity index 67% rename from Quorum/entry_points/ipfs_validator.py rename to Quorum/entry_points/implementations/ipfs_validator.py index d2c21f6..d076e99 100644 --- a/Quorum/entry_points/ipfs_validator.py +++ b/Quorum/entry_points/implementations/ipfs_validator.py @@ -1,36 +1,17 @@ -from Quorum.utils.chain_enum import Chain +import argparse +import requests +from pathlib import Path + from Quorum.apis.block_explorers.chains_api import ChainAPI from Quorum.llm.chains.ipfs_validation_chain import IPFSValidationChain import Quorum.utils.config as config -import Quorum.utils.arg_validations as arg_valid import Quorum.utils.pretty_printer as pp -from pathlib import Path -import argparse -import requests - IPFS_CACHE = Path(__file__).parent / '.ipfs_cache' IPFS_CACHE.mkdir(exist_ok=True) -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description='Compare ipfs with actual payload.') - parser.add_argument('--proposal_id', type=int, help='The id of the proposal.') - parser.add_argument('--chain', type=str, choices=[chain.value for chain in Chain], help='Blockchain chain.') - parser.add_argument('--proposal_address', type=arg_valid.validate_address, help='Ethereum proposal address.') - parser.add_argument( - '--prompt_templates', - type=str, - nargs="+", - default=['ipfs_validation_prompt_part1.j2', "ipfs_validation_prompt_part2.j2"], - help='Jinja templates for prompting the LLM.' - ) - - args = parser.parse_args() - - return args - def get_raw_ipfs(proposal_id: int) -> str: cache = IPFS_CACHE / f'{proposal_id}.txt' if cache.exists(): @@ -50,11 +31,31 @@ def get_raw_ipfs(proposal_id: int) -> str: return ipfs_content -def main(): +def run_ipfs_validator(args: argparse.Namespace): + """ + Validates IPFS content against proposal source code using LLM-based analysis. + This function performs validation between IPFS content and smart contract source code + by leveraging Language Model analysis through a validation chain. + Args: + args (argparse.Namespace): Command line arguments containing: + - chain: The blockchain network to query + - proposal_address: Contract address of the proposal + - proposal_id: IPFS identifier for the proposal + - prompt_templates: Templates for LLM prompts + Raises: + ValueError: If ANTHROPIC_API_KEY is not set in environment variables + ValueError: If no source code is found for the given proposal address + Returns: + None. Results are printed to stdout: + - Lists incompatibilities if found + - Warning message if no incompatibilities detected + Example: + args = parser.parse_args() + run_ipfs_validator(args) + """ # Check if the Anthropic API key is set in environment variables if not config.ANTHROPIC_API_KEY: raise ValueError("ANTHROPIC_API_KEY environment variable is not set. Please set it to use this functionality.") - args = parse_args() # Initialize Chain API and fetch source codes block_explorer = ChainAPI(args.chain) @@ -80,6 +81,3 @@ def main(): pp.pretty_print(incompatibility, pp.Colors.FAILURE) else: pp.pretty_print("LLM found no incompatibilities. Please Check manually.", pp.Colors.WARNING) - -if __name__ == '__main__': - main() diff --git a/Quorum/entry_points/setup_quorum.py b/Quorum/entry_points/implementations/setup_quorum.py similarity index 55% rename from Quorum/entry_points/setup_quorum.py rename to Quorum/entry_points/implementations/setup_quorum.py index 0ee3077..340d75c 100644 --- a/Quorum/entry_points/setup_quorum.py +++ b/Quorum/entry_points/implementations/setup_quorum.py @@ -4,32 +4,31 @@ import Quorum.utils.pretty_printer as pp - -def get_working_directory() -> Path: - parser = argparse.ArgumentParser(description="Setup Quorum project.") - parser.add_argument( - '--working_dir', - default=Path.cwd() / 'quorum_project', - type=Path, - help="Directory to set up the Quorum project." - ) - args = parser.parse_args() - return args.working_dir - - -def setup_quorum(working_dir: Path): +def run_setup_quorum(args: argparse.Namespace): """ - Initializes Quorum environment by copying template files to the specified directory. - + Sets up a new Quorum working directory with template files and environment configuration. + This function creates a new directory (if it doesn't exist) and populates it with required + template files for Quorum operation. It also configures the environment variables. Args: - working_dir (Path): Target directory for setting up Quorum. - + args (argparse.Namespace): Command line arguments containing: + - working_dir: Path object specifying target directory for setup + Template files copied: + - .env.example -> .env + - execution.json + - ground_truth.json + - README.md + The function will: + 1. Create target directory if it doesn't exist + 2. Copy template files, skipping any that already exist + 3. Add QUORUM_PATH export to .env file + Returns: + None Raises: - shutil.Error: If copying files fails. - OSError: If directory creation fails. + OSError: If there are filesystem permission issues + shutil.Error: If file copy operations fail """ templates_dir = Path(__file__).parent.parent / 'templates' - target_dir = working_dir.resolve() + target_dir = args.working_dir.resolve() if not target_dir.exists(): pp.pretty_print(f"Creating directory: {target_dir}", pp.Colors.INFO) @@ -54,16 +53,3 @@ def setup_quorum(working_dir: Path): f.write(f'\nexport QUORUM_PATH="{target_dir}"\n') pp.pretty_print("Quorum setup completed successfully!", pp.Colors.SUCCESS) - - -def main(): - working_dir = get_working_directory() - try: - setup_quorum(working_dir) - except Exception as e: - pp.pretty_print(f"Setup failed: {e}", pp.Colors.FAILURE) - exit(1) - - -if __name__ == "__main__": - main() diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py new file mode 100644 index 0000000..d3a5349 --- /dev/null +++ b/Quorum/entry_points/quorum_cli.py @@ -0,0 +1,107 @@ +# Quorum/entry_points/quorum_cli.py + +import argparse + +import Quorum.entry_points.cli_arguments as cli_args +from Quorum.entry_points.implementations.check_proposal import run_single +from Quorum.entry_points.implementations.check_proposal_config import run_config +from Quorum.entry_points.implementations.check_proposal_id import run_proposal_id +from Quorum.entry_points.implementations.create_report import run_create_report +from Quorum.entry_points.implementations.ipfs_validator import run_ipfs_validator +from Quorum.entry_points.implementations.setup_quorum import run_setup_quorum + + +# Define a registry of subcommands +COMMAND_REGISTRY = [ + { + "name": "single-payload", + "help": "Run a single payload proposal check.", + "arguments": [ + cli_args.CUSTOMER_ARGUMENT, + cli_args.CHAIN_ARGUMENT, + cli_args.PROPOSAL_ADDRESS_ARGUMENT + ], + "func": run_single + }, + { + "name": "config", + "help": "Run a batch check from a JSON config file.", + "arguments": [cli_args.CONFIG_ARGUMENT], + "func": run_config + }, + { + "name": "proposal-id", + "help": "Check proposals by proposal ID.", + "arguments": [ + cli_args.CUSTOMER_ARGUMENT, + cli_args.PROPOSAL_ID_ARGUMENT + ], + "func": run_proposal_id + }, + { + "name": "create-report", + "help": "Generate a proposal report.", + "arguments": [ + cli_args.PROPOSAL_ID_ARGUMENT, + cli_args.TEMPLATE_ARGUMENT, + cli_args.GENERATE_REPORT_PATH_ARGUMENT + ], + "func": run_create_report + }, + { + "name": "ipfs-validate", + "help": "Compare IPFS content with a proposal’s payload.", + "arguments": [ + cli_args.PROPOSAL_ID_ARGUMENT, + cli_args.CHAIN_ARGUMENT, + cli_args.PROPOSAL_ADDRESS_ARGUMENT, + cli_args.PROMPT_TEMPLATES_ARGUMENT + ], + "func": run_ipfs_validator + }, + { + "name": "setup", + "help": "Initial Quorum environment setup.", + "arguments": [cli_args.WORKING_DIR_ARGUMENT], + "func": run_setup_quorum + } + ] + + +def add_arguments(parser: argparse.ArgumentParser, arguments: list[cli_args.Argument]) -> None: + """ + Helper function to add arguments to a parser. + + Args: + parser (argparse.ArgumentParser): The parser or subparser to add arguments to. + arguments (List[Argument]): A list of Argument instances to add. + """ + for arg in arguments: + parser.add_argument(**arg.model_dump()) + + +def main(): + parser = argparse.ArgumentParser( + prog="Quorum", + description="Main Quorum CLI with multiple subcommands." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + + # Iterate over the registry to add subcommands + for subcmd in COMMAND_REGISTRY: + subparser = subparsers.add_parser( + subcmd["name"], + help=subcmd["help"] + ) + add_arguments(subparser, subcmd["arguments"]) + subparser.set_defaults(func=subcmd["func"]) + + args = parser.parse_args() + + # Dispatch to the appropriate function + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/Quorum/tests/test_auto_report.py b/Quorum/tests/test_auto_report.py index 358666a..a2902ec 100644 --- a/Quorum/tests/test_auto_report.py +++ b/Quorum/tests/test_auto_report.py @@ -14,7 +14,7 @@ def test_auto_report(): with open(EXPECTED_DIR / 'v3-132.md') as f: expected = f.read() - create_report.create_report(132, create_report.DEFAULT_TEMPLATE_PATH, Path('v3-132.md')) + create_report.run_create_report(132, create_report.DEFAULT_TEMPLATE_PATH, Path('v3-132.md')) with open(Path('v3-132.md')) as f: actual = f.read() diff --git a/setup.py b/setup.py index ad55f70..83398a0 100644 --- a/setup.py +++ b/setup.py @@ -19,12 +19,7 @@ def read_version() -> str: }, entry_points={ "console_scripts": [ - "CheckProposal=Quorum.entry_points.check_proposal:main", - "CheckProposalConfig=Quorum.entry_points.check_proposal_config:main", - "CheckProposalId=Quorum.entry_points.check_proposal_id:main", - "IPFSValidator=Quorum.entry_points.ipfs_validator:main", - "CreateReport=Quorum.entry_points.create_report:main", - "SetupQuorum=Quorum.entry_points.setup_quorum:main", + "Quorum=Quorum.entry_points.quorum_cli:main", ], }, ) diff --git a/version b/version index d8f13c1..7af46ef 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250105.192006.194722 +20250106.124431.181672 From cb40c8cbad7f33555f31d135352a9f7032eda479 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 12:54:22 +0200 Subject: [PATCH 10/24] fix parse + CI --- .github/workflows/CI.yml | 6 +++--- Quorum/entry_points/quorum_cli.py | 4 +++- version | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5fb48e8..1c6ed3b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,6 +39,6 @@ jobs: - name: Execute Regression Tests run: | pytest Quorum/tests --maxfail=1 --disable-warnings --tb=short - CheckProposal --customer Aave --chain Ethereum --proposal_address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637 - CheckProposalConfig --config Quorum/tests/regression.json - CheckProposalId --proposal_id 137 --customer Aave + Quroum single-payload --customer Aave --chain Ethereum --proposal_address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637 + Quroum config --config Quorum/tests/regression.json + Quroum proposal-id --proposal_id 137 --customer Aave diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py index d3a5349..e27f739 100644 --- a/Quorum/entry_points/quorum_cli.py +++ b/Quorum/entry_points/quorum_cli.py @@ -77,7 +77,9 @@ def add_arguments(parser: argparse.ArgumentParser, arguments: list[cli_args.Argu arguments (List[Argument]): A list of Argument instances to add. """ for arg in arguments: - parser.add_argument(**arg.model_dump()) + arg_dict = arg.model_dump() + name = arg_dict.pop("name") + parser.add_argument(name, **arg_dict) def main(): diff --git a/version b/version index 7af46ef..f1bbc15 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.124431.181672 +20250106.125422.591891 From 8288e48f22d262c2c153f19d7579bd2f1342b7db Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 13:01:05 +0200 Subject: [PATCH 11/24] . --- pyproject.toml | 5 +---- version | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 version diff --git a/pyproject.toml b/pyproject.toml index e1530a3..2346682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,7 @@ Homepage = "https://github.com/Certora/Quorum" Source = "https://github.com/Certora/Quorum" [project.scripts] -CheckProposal = "Quorum.entry_points.check_proposal:main" -IPFSValidator = "Quorum.entry_points.ipfs_validator:main" -CreateReport = "Quorum.entry_points.create_report:main" -SetupQuorum = "Quorum.entry_points.setup_quorum:main" +Quorum = "Quorum.entry_points.quorum_cli:main" [tool.setuptools] include-package-data = true diff --git a/version b/version new file mode 100644 index 0000000..801cffe --- /dev/null +++ b/version @@ -0,0 +1 @@ +20250106.130106.088745 From 72c247e0f1f6ebf7c4462f2e1f16a901c808d534 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 14:10:06 +0200 Subject: [PATCH 12/24] Fix test --- Quorum/tests/test_auto_report.py | 11 +++++++++-- version | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Quorum/tests/test_auto_report.py b/Quorum/tests/test_auto_report.py index a2902ec..0571f23 100644 --- a/Quorum/tests/test_auto_report.py +++ b/Quorum/tests/test_auto_report.py @@ -1,8 +1,9 @@ import pytest +import argparse import Quorum.tests.conftest as conftest -import Quorum.entry_points.create_report as create_report +import Quorum.entry_points.implementations.create_report as create_report from pathlib import Path @@ -13,8 +14,14 @@ def test_auto_report(): with open(EXPECTED_DIR / 'v3-132.md') as f: expected = f.read() + + args = argparse.Namespace( + proposal_id=132, + template=Path('Quorum') / 'auto_report' / 'AaveReportTemplate.md.j2', + generate_report_path=None + ) - create_report.run_create_report(132, create_report.DEFAULT_TEMPLATE_PATH, Path('v3-132.md')) + create_report.run_create_report(args) with open(Path('v3-132.md')) as f: actual = f.read() diff --git a/version b/version index 801cffe..30df66b 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.130106.088745 +20250106.141006.581797 From 8f09ecae6e4b69a841ef401229747d6cbd187a34 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 14:54:10 +0200 Subject: [PATCH 13/24] Include Quroum package at install --- pyproject.toml | 3 +++ version | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2346682..9e90fd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,3 +32,6 @@ Quorum = "Quorum.entry_points.quorum_cli:main" [tool.setuptools] include-package-data = true + +[tool.setuptools.packages.find] +include = ["Quorum"] diff --git a/version b/version index 30df66b..31d059d 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.141006.581797 +20250106.145410.953816 From d57485bc627138b0d3df3bbdf5b7809501aff59a Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 15:15:35 +0200 Subject: [PATCH 14/24] Fix ci --- .github/workflows/CI.yml | 6 +++--- version | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1c6ed3b..692d377 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,6 +39,6 @@ jobs: - name: Execute Regression Tests run: | pytest Quorum/tests --maxfail=1 --disable-warnings --tb=short - Quroum single-payload --customer Aave --chain Ethereum --proposal_address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637 - Quroum config --config Quorum/tests/regression.json - Quroum proposal-id --proposal_id 137 --customer Aave + Quorum single-payload --customer Aave --chain Ethereum --proposal_address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637 + Quorum config --config Quorum/tests/regression.json + Quorum proposal-id --proposal_id 137 --customer Aave diff --git a/version b/version index 31d059d..acca518 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.145410.953816 +20250106.151536.447335 From a51df2c4ef68c32d0d2421c2f15dd1bff9e07c6b Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 15:32:33 +0200 Subject: [PATCH 15/24] Try now --- pyproject.toml | 3 --- version | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e90fd6..2346682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,3 @@ Quorum = "Quorum.entry_points.quorum_cli:main" [tool.setuptools] include-package-data = true - -[tool.setuptools.packages.find] -include = ["Quorum"] diff --git a/version b/version index acca518..fb953bc 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.151536.447335 +20250106.153234.318956 From 7691e4845f7227bafe71e9d0baffeed37a88fa45 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:09:05 +0200 Subject: [PATCH 16/24] Address Review comments --- Quorum/apis/governance/data_models.py | 24 ++-- Quorum/auto_report/aave_tags.py | 12 +- Quorum/checks/proposal_check.py | 51 ++++++++ .../implementations/check_proposal.py | 56 ++------ .../implementations/check_proposal_config.py | 64 ++------- .../implementations/check_proposal_id.py | 58 ++------- Quorum/entry_points/quorum_cli.py | 123 +++++++++--------- .../tests/expected/test_auto_report/v3-132.md | 5 +- version | 2 +- 9 files changed, 174 insertions(+), 221 deletions(-) diff --git a/Quorum/apis/governance/data_models.py b/Quorum/apis/governance/data_models.py index 72536d2..8fb16de 100644 --- a/Quorum/apis/governance/data_models.py +++ b/Quorum/apis/governance/data_models.py @@ -3,9 +3,10 @@ from Quorum.utils.chain_enum import Chain + class IPFSData(BaseModel): - title: Optional[str] = None - discussions: Optional[str] = None + title: str = 'N/A' + discussions: str = 'N/A' class PayloadData(BaseModel): @@ -19,18 +20,25 @@ class Config: class ProposalData(BaseModel): payloads: list[PayloadData] = Field(default_factory=list) - votingPortal: Optional[str] = None - ipfsHash: Optional[str] = None + voting_portal: Optional[str] = Field(alias='votingPortal') + ipfs_hash: Optional[str] = Field(alias='ipfsHash') + access_level: Optional[str | int] = Field(alias='accessLevel') + + class Config: + allow_population_by_alias = True class EventArgs(BaseModel): - creator: Optional[str] = None - accessLevel: Optional[int] = None - ipfsHash: Optional[str] = None + creator: str = 'N/A' + access_level: Optional[str | int] = Field(alias='accessLevel', default='N/A') + ipfs_hash: str = Field(alias='ipfsHash', default='N/A') + + class Config: + allow_population_by_alias = True class EventData(BaseModel): - transactionHash: Optional[str] = None + transaction_hash: str = Field(alias='transactionHash') args: EventArgs = Field(default_factory=EventArgs) diff --git a/Quorum/auto_report/aave_tags.py b/Quorum/auto_report/aave_tags.py index 586453d..50a200e 100644 --- a/Quorum/auto_report/aave_tags.py +++ b/Quorum/auto_report/aave_tags.py @@ -52,9 +52,9 @@ def get_aave_tags(proposal_id: int) -> Dict[str, Any]: # Basic info tags['proposal_id'] = str(proposal_id) - tags['proposal_title'] = ipfs_data.title if ipfs_data.title else 'N/A' + tags['proposal_title'] = ipfs_data.title tags['voting_link'] = f'https://vote.onaave.com/proposal/?proposalId={proposal_id}' - tags['gov_forum_link'] = ipfs_data.discussions if ipfs_data.discussions else 'N/A' + tags['gov_forum_link'] = ipfs_data.discussions # Multi-chain references tags['chain'] = [] @@ -87,15 +87,15 @@ def get_aave_tags(proposal_id: int) -> Dict[str, Any]: tags['payload_seatbelt_link'].append(seatbelt_link) # Transaction info - transaction_hash = create_event.transactionHash or 'N/A' + transaction_hash = create_event.transaction_hash tags['transaction_hash'] = transaction_hash tags['transaction_link'] = f'https://etherscan.io/tx/{transaction_hash}' # Creator + event args args = create_event.args - tags['creator'] = args.creator if args.creator else 'N/A' - tags['access_level'] = str(args.accessLevel) if args.accessLevel is not None else 'N/A' - tags['ipfs_hash'] = args.ipfsHash if args.ipfsHash else 'N/A' + tags['creator'] = args.creator + tags['access_level'] = args.access_level + tags['ipfs_hash'] = args.ipfs_hash tags['createProposal_parameters_data'] = json.dumps(proposal_data.model_dump(), indent=4) diff --git a/Quorum/checks/proposal_check.py b/Quorum/checks/proposal_check.py index cdb097d..2436f36 100644 --- a/Quorum/checks/proposal_check.py +++ b/Quorum/checks/proposal_check.py @@ -1,8 +1,59 @@ +from pydantic import BaseModel + import Quorum.checks as Checks import Quorum.utils.pretty_printer as pp from Quorum.utils.chain_enum import Chain from Quorum.apis.block_explorers.chains_api import ChainAPI from Quorum.apis.price_feeds.price_feed_utils import PriceFeedProviderBase +from Quorum.apis.governance.data_models import PayloadAddresses +from Quorum.apis.git_api.git_manager import GitManager +import Quorum.utils.config_loader as ConfigLoader + + + +class CustomerConfig(BaseModel): + customer: str + payload_addresses: list[PayloadAddresses] + + +class ProposalConfig(BaseModel): + customers_config: list[CustomerConfig] + + +def run_customer_proposal_validation(prop_config: ProposalConfig) -> None: + """ + Execute proposal checks in batch for multiple customers and their configurations. + + This function processes proposal configurations for multiple customers, clones or updates + their repositories, and performs proposal checks for specified addresses on different chains. + + Args: + prop_config (ProposalConfig): Configuration object containing customer configs, + payload addresses, and chain information for proposal validation. + + Returns: + None + + Example: + >>> prop_config = ProposalConfig(...) + >>> run_batch(prop_config) + """ + for config in prop_config.customers_config: + ground_truth_config = ConfigLoader.load_customer_config(config.customer) + + git_manager = GitManager(config.customer, ground_truth_config) + git_manager.clone_or_update() + + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + + for pa in config.payload_addresses: + proposals_check( + customer=config.customer, + chain=pa.chain, + proposal_addresses=pa.addresses, + providers=price_feed_providers + ) + def proposals_check(customer: str, chain: Chain, proposal_addresses: list[str], providers: list[PriceFeedProviderBase]) -> None: diff --git a/Quorum/entry_points/implementations/check_proposal.py b/Quorum/entry_points/implementations/check_proposal.py index c8a8a39..b0edd26 100644 --- a/Quorum/entry_points/implementations/check_proposal.py +++ b/Quorum/entry_points/implementations/check_proposal.py @@ -1,56 +1,22 @@ import argparse -from Quorum.apis.git_api.git_manager import GitManager -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader +from Quorum.checks.proposal_check import run_customer_proposal_validation, ProposalConfig, CustomerConfig, PayloadAddresses + def run_single(args: argparse.Namespace) -> None: """ - Execute proposal verification for a single proposal address. - - This function processes a single proposal check by initializing necessary configurations - and git repositories, then performs validation checks on the specified proposal. + Run a single proposal check for a specific customer and chain. Args: args (argparse.Namespace): Command line arguments containing: - - customer (str): Customer identifier - - chain (str): Blockchain network identifier - - proposal_address (str): Address of the proposal to check + - customer (str): The customer identifier + - chain (str): The blockchain network identifier + - proposal_address (str): The address of the proposal to check - Raises: - ValueError: If any of the required arguments (customer, chain, proposal_address) are missing - - Example: - args = argparse.Namespace( - customer='example_customer', - chain='ethereum', - proposal_address='0x1234...' - run_single(args) + The function creates customer and proposal configurations based on the input arguments + and executes a batch run for the single proposal. """ - # Parse command-line arguments customer, chain, proposal_address = args.customer, args.chain, args.proposal_address - - # Ensure all required arguments are provided - if not (customer and chain and proposal_address): - raise ValueError( - "Customer, chain, and proposal_address must be specified." - " Provide all three arguments when not using a config file." - ) - - # Load the customer's ground truth configuration - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Execute proposal checks - proposals_check( - customer=customer, - chain=chain, - proposal_addresses=[proposal_address], - providers=price_feed_providers - ) + customer_config = CustomerConfig(customer=customer, payload_addresses=[PayloadAddresses(chain=chain, addresses=[proposal_address])]) + prop_config = ProposalConfig(customers_config=[customer_config]) + run_customer_proposal_validation(prop_config) diff --git a/Quorum/entry_points/implementations/check_proposal_config.py b/Quorum/entry_points/implementations/check_proposal_config.py index cf1b47a..8d0fff8 100644 --- a/Quorum/entry_points/implementations/check_proposal_config.py +++ b/Quorum/entry_points/implementations/check_proposal_config.py @@ -1,71 +1,27 @@ import argparse from typing import Dict, Any -from Quorum.apis.git_api.git_manager import GitManager -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader -from Quorum.utils.chain_enum import Chain +from Quorum.checks.proposal_check import run_customer_proposal_validation, ProposalConfig def run_config(args: argparse.Namespace) -> None: """ - Processes and validates proposal configurations for different customers across blockchain networks. + Execute configuration for proposal validation. - This function takes command-line arguments containing configuration data and performs the following: - - Loads ground truth configurations for each customer - - Manages Git repositories through GitManager - - Validates blockchain chains - - Executes proposal checks for specified addresses + This function takes command line arguments containing configuration data, + creates a ProposalConfig object, and runs batch processing with the configuration. Args: - args (argparse.Namespace): Command-line arguments containing the configuration data - in args.config as a dictionary + args (argparse.Namespace): Command line arguments containing the config data. + Expected to have a 'config' attribute with proposal + configuration dictionary. Returns: None Raises: - ValueError: If an unsupported or invalid blockchain chain is specified - - Example config_data structure: - { - "customer_name": { - "chain_name": { - "Proposals": ["0x123...", "0x456..."] - } - } - } + TypeError: If config_data cannot be unpacked into ProposalConfig """ - # Parse command-line arguments to get the configuration data config_data: Dict[str, Any] = args.config - - # Iterate over each customer and their respective chain and proposal information in the config - for customer, chain_info in config_data.items(): - # Load the ground truth configuration for the specified customer - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Iterate over each blockchain chain and its associated proposals for the customer - for chain, proposals in chain_info.items(): - try: - # Validate and convert the chain identifier to the Chain enum - chain_enum = Chain(chain) - except ValueError as e: - # Handle unsupported or invalid chain identifiers - raise ValueError(f"Unsupported or invalid chain '{chain}' for customer '{customer}': {e}") - - # Check if there are any proposals to process for the current chain - if proposals.get("Proposals"): - # Execute proposal checks for the specified customer, chain, and proposal addresses - proposals_check( - customer=customer, - chain=chain_enum, - proposal_addresses=proposals["Proposals"], - providers=price_feed_providers - ) + prop_config = ProposalConfig(**config_data) + run_customer_proposal_validation(prop_config) diff --git a/Quorum/entry_points/implementations/check_proposal_id.py b/Quorum/entry_points/implementations/check_proposal_id.py index 6f00981..a18814a 100644 --- a/Quorum/entry_points/implementations/check_proposal_id.py +++ b/Quorum/entry_points/implementations/check_proposal_id.py @@ -1,9 +1,7 @@ import argparse -from Quorum.apis.git_api.git_manager import GitManager from Quorum.apis.governance.aave_governance import AaveGovernanceAPI -from Quorum.checks.proposal_check import proposals_check -import Quorum.utils.config_loader as ConfigLoader +from Quorum.checks.proposal_check import run_customer_proposal_validation, ProposalConfig CUSTOMER_TO_API = { @@ -13,63 +11,29 @@ def run_proposal_id(args: argparse.Namespace) -> None: """ - Executes a validation process for a given proposal ID associated with a specific customer. + Executes proposal validation for a specific customer and proposal ID. - This function performs several key operations: - 1. Validates the customer and retrieves their governance API - 2. Fetches payload addresses for the given proposal ID - 3. Loads customer configuration and sets up git repositories - 4. Performs proposal checks for each payload + This function retrieves payload addresses for a given proposal and runs batch validation + using the customer's API configuration. Args: args (argparse.Namespace): Command line arguments containing: - - customer (str): The identifier of the customer - - proposal_id (str/int): The ID of the proposal to check + - customer (str): Name of the customer to validate proposal for + - proposal_id (str/int): ID of the proposal to validate Raises: - ValueError: If the specified customer is not supported + ValueError: If the provided customer is not supported in CUSTOMER_TO_API mapping - Example: - args = argparse.Namespace(customer='example', proposal_id='123') - run_proposal_id(args) + Returns: + None """ - - # Parse command-line arguments customer, proposal_id = args.customer, args.proposal_id - - # Normalize customer identifier to lowercase for consistent mapping customer_key = customer.lower() - - # Validate if the specified customer is supported if customer_key not in CUSTOMER_TO_API: raise ValueError(f"Customer '{customer}' is not supported. Supported customers: {list(CUSTOMER_TO_API.keys())}.") - # Retrieve the appropriate governance API instance for the customer api = CUSTOMER_TO_API[customer_key] - - # Fetch all payload addresses associated with the given proposal ID payloads_addresses = api.get_all_payloads_addresses(proposal_id) + config = ProposalConfig(customers_config=[{"customer": customer, "payload_addresses": payloads_addresses}]) - # Load the ground truth configuration for the specified customer - ground_truth_config = ConfigLoader.load_customer_config(customer) - - # Initialize GitManager with customer and configuration, then clone or update repositories - git_manager = GitManager(customer, ground_truth_config) - git_manager.clone_or_update() - - # Retrieve price feed providers from the configuration, defaulting to an empty list if not specified - price_feed_providers = ground_truth_config.get("price_feed_providers", []) - - # Iterate over each payload's data and perform proposal checks - for payload_data in payloads_addresses: - """ - payload_data is expected to be an object with at least the following attributes: - - chain: The blockchain chain associated with the payload. - - addresses: A list of smart contract addresses associated with the payload. - """ - proposals_check( - customer=customer, - chain=payload_data.chain, - proposal_addresses=payload_data.addresses, - providers=price_feed_providers - ) + run_customer_proposal_validation(config) diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py index e27f739..21b98b2 100644 --- a/Quorum/entry_points/quorum_cli.py +++ b/Quorum/entry_points/quorum_cli.py @@ -1,6 +1,7 @@ # Quorum/entry_points/quorum_cli.py import argparse +from pydantic import BaseModel import Quorum.entry_points.cli_arguments as cli_args from Quorum.entry_points.implementations.check_proposal import run_single @@ -11,61 +12,67 @@ from Quorum.entry_points.implementations.setup_quorum import run_setup_quorum -# Define a registry of subcommands +class Command(BaseModel): + name: str + help: str + arguments: list[cli_args.Argument] + func: callable + + COMMAND_REGISTRY = [ - { - "name": "single-payload", - "help": "Run a single payload proposal check.", - "arguments": [ - cli_args.CUSTOMER_ARGUMENT, - cli_args.CHAIN_ARGUMENT, - cli_args.PROPOSAL_ADDRESS_ARGUMENT - ], - "func": run_single - }, - { - "name": "config", - "help": "Run a batch check from a JSON config file.", - "arguments": [cli_args.CONFIG_ARGUMENT], - "func": run_config - }, - { - "name": "proposal-id", - "help": "Check proposals by proposal ID.", - "arguments": [ - cli_args.CUSTOMER_ARGUMENT, - cli_args.PROPOSAL_ID_ARGUMENT - ], - "func": run_proposal_id - }, - { - "name": "create-report", - "help": "Generate a proposal report.", - "arguments": [ - cli_args.PROPOSAL_ID_ARGUMENT, - cli_args.TEMPLATE_ARGUMENT, - cli_args.GENERATE_REPORT_PATH_ARGUMENT - ], - "func": run_create_report - }, - { - "name": "ipfs-validate", - "help": "Compare IPFS content with a proposal’s payload.", - "arguments": [ - cli_args.PROPOSAL_ID_ARGUMENT, - cli_args.CHAIN_ARGUMENT, - cli_args.PROPOSAL_ADDRESS_ARGUMENT, - cli_args.PROMPT_TEMPLATES_ARGUMENT - ], - "func": run_ipfs_validator - }, - { - "name": "setup", - "help": "Initial Quorum environment setup.", - "arguments": [cli_args.WORKING_DIR_ARGUMENT], - "func": run_setup_quorum - } - ] + Command( + name="single-payload", + help="Run a single payload proposal check.", + arguments=[ + cli_args.CUSTOMER_ARGUMENT, + cli_args.CHAIN_ARGUMENT, + cli_args.PROPOSAL_ADDRESS_ARGUMENT + ], + func=run_single + ), + Command( + name="config", + help="Run a batch check from a JSON config file.", + arguments=[cli_args.CONFIG_ARGUMENT], + func=run_config + ), + Command( + name="proposal-id", + help="Check proposals by proposal ID.", + arguments=[ + cli_args.CUSTOMER_ARGUMENT, + cli_args.PROPOSAL_ID_ARGUMENT + ], + func=run_proposal_id + ), + Command( + name="create-report", + help="Generate a proposal report.", + arguments=[ + cli_args.PROPOSAL_ID_ARGUMENT, + cli_args.TEMPLATE_ARGUMENT, + cli_args.GENERATE_REPORT_PATH_ARGUMENT + ], + func=run_create_report + ), + Command( + name="ipfs-validate", + help="Compare IPFS content with a proposal's payload.", + arguments=[ + cli_args.PROPOSAL_ID_ARGUMENT, + cli_args.CHAIN_ARGUMENT, + cli_args.PROPOSAL_ADDRESS_ARGUMENT, + cli_args.PROMPT_TEMPLATES_ARGUMENT + ], + func=run_ipfs_validator + ), + Command( + name="setup", + help="Initial Quorum environment setup.", + arguments=[cli_args.WORKING_DIR_ARGUMENT], + func=run_setup_quorum + ) +] def add_arguments(parser: argparse.ArgumentParser, arguments: list[cli_args.Argument]) -> None: @@ -93,11 +100,11 @@ def main(): # Iterate over the registry to add subcommands for subcmd in COMMAND_REGISTRY: subparser = subparsers.add_parser( - subcmd["name"], - help=subcmd["help"] + subcmd.name, + help=subcmd.help ) - add_arguments(subparser, subcmd["arguments"]) - subparser.set_defaults(func=subcmd["func"]) + add_arguments(subparser, subcmd.arguments) + subparser.set_defaults(func=subcmd.func) args = parser.parse_args() diff --git a/Quorum/tests/expected/test_auto_report/v3-132.md b/Quorum/tests/expected/test_auto_report/v3-132.md index ee661f9..6411538 100644 --- a/Quorum/tests/expected/test_auto_report/v3-132.md +++ b/Quorum/tests/expected/test_auto_report/v3-132.md @@ -106,8 +106,9 @@ Transaction: [0x423b2b381444d3a8a347536eaf643da3c7bc5e764ff4881863e012305d9542ba payload_id: 17, }, ], - votingPortal: "0x9b24C168d6A76b5459B1d47071a54962a4df36c3", - ipfsHash: "0x392c2cdfd6c2f57a7be73b170d472b4b8e6c662cb941451b449a0b2988ab3d57", + voting_portal: "0x9b24C168d6A76b5459B1d47071a54962a4df36c3", + ipfs_hash: "0x392c2cdfd6c2f57a7be73b170d472b4b8e6c662cb941451b449a0b2988ab3d57", + access_level: 1, } ``` diff --git a/version b/version index fb953bc..a689893 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.153234.318956 +20250106.170905.948471 From 4097d3d33e0e9b117afef50fb063d18e3568e11a Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:13:12 +0200 Subject: [PATCH 17/24] fix --- Quorum/entry_points/quorum_cli.py | 3 +++ version | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py index 21b98b2..716a5b9 100644 --- a/Quorum/entry_points/quorum_cli.py +++ b/Quorum/entry_points/quorum_cli.py @@ -18,6 +18,9 @@ class Command(BaseModel): arguments: list[cli_args.Argument] func: callable + class Config: + arbitrary_types_allowed = True + COMMAND_REGISTRY = [ Command( diff --git a/version b/version index a689893..534a79b 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.170905.948471 +20250106.171312.982801 From 20a4b3cba7e65dab7361f6cf08dc09845bd8709b Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:17:30 +0200 Subject: [PATCH 18/24] Fix typing --- Quorum/entry_points/quorum_cli.py | 6 ++---- version | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py index 716a5b9..c0f260b 100644 --- a/Quorum/entry_points/quorum_cli.py +++ b/Quorum/entry_points/quorum_cli.py @@ -2,6 +2,7 @@ import argparse from pydantic import BaseModel +from typing import Callable import Quorum.entry_points.cli_arguments as cli_args from Quorum.entry_points.implementations.check_proposal import run_single @@ -16,10 +17,7 @@ class Command(BaseModel): name: str help: str arguments: list[cli_args.Argument] - func: callable - - class Config: - arbitrary_types_allowed = True + func: Callable[[argparse.Namespace], None] COMMAND_REGISTRY = [ diff --git a/version b/version index 534a79b..73defc9 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.171312.982801 +20250106.171730.755013 From 218e6df12b795495a83bc5d761f52702ac464d71 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:26:41 +0200 Subject: [PATCH 19/24] Update README file --- README.md | 41 +++++++++++++++++++++-------------------- version | 2 +- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e72c533..52ca59e 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,14 @@ Quorum offers a convenient setup command to streamline initial configuration by ### 1. Run Setup Command ```bash -SetupQuorum [working_directory] +Quorum setup --working_dir "/home/user/quorum_project" ``` - **`working_directory`** *(Optional)*: Path where Quorum’s configuration files will be placed. If omitted, the current directory is used. **Example**: ```bash -SetupQuorum ./my_quorum_project +Quorum setup --working_dir ./my_quorum_project ``` This action will: @@ -131,57 +131,57 @@ To fully enable Quorum’s checks, set the following: ## Usage -Quorum provides multiple entry points to handle different aspects of proposal checks and code validation. Below is an overview of each command, with examples. +Quorum now provides a **single CLI** with multiple **subcommands** for different tasks. Below is an overview of each subcommand, with examples. -### 1. **CheckProposal** +### 1. **single-payload** -**Purpose:** Analyzes a single proposal by address for a specific customer on a given chain. +**Purpose:** Analyzes a single proposal address for a specific customer on a given chain. ```bash -CheckProposal --customer "Aave" --chain "Ethereum" --proposal_address "0xAD6..." +Quorum single-payload --customer "Aave" --chain "Ethereum" --proposal_address "0xAD6..." ``` -### 2. **CheckProposalConfig** +### 2. **config** **Purpose:** Processes multiple proposals in bulk using a JSON config file. ```bash -CheckProposalConfig --config "/path/to/config.json" +Quorum config --config "/path/to/config.json" ``` *(See “**Example Usage with Config File**” for a sample config.)* -### 3. **CheckProposalId** +### 3. **proposal-id** **Purpose:** Looks up all payload addresses for a single proposal ID (useful for proposals containing multiple payloads). ```bash -CheckProposalId --customer "Aave" --proposal_id 137 +Quorum proposal-id --customer "Aave" --proposal_id 137 ``` -### 4. **IPFSValidator** +### 4. **ipfs-validate** **Purpose:** Validates whether the IPFS description content aligns with the actual on-chain payload. Uses LLM-based analysis. ```bash -IPFSValidator --proposal_id 132 --chain "Ethereum" --proposal_address "0xAD6..." +Quorum ipfs-validate --proposal_id 132 --chain "Ethereum" --proposal_address "0xAD6..." ``` -### 5. **CreateReport** +### 5. **create-report** **Purpose:** Generates a human-readable report of the proposal details, leveraging Jinja2 templates. ```bash -CreateReport --proposal_id 137 \ - --template "Quorum/auto_report/AaveReportTemplate.md.j2" \ - --generate_report_path "reports/v3-137.md" +Quorum create-report --proposal_id 137 \ + --template "Quorum/auto_report/AaveReportTemplate.md.j2" \ + --generate_report_path "reports/v3-137.md" ``` -### 6. **SetupQuorum** +### 6. **setup** **Purpose:** Bootstraps your Quorum environment, creating `.env`, `ground_truth.json`, `execution.json`, and an initial `README.md`. ```bash -SetupQuorum --working_dir "/home/user/quorum_project" +Quorum setup --working_dir "/home/user/quorum_project" ``` *(Refer to “**Quick Setup**” for details.)* @@ -205,7 +205,7 @@ For bulk execution, create a config file (e.g., `config.json`) with the followin Then run: ```bash -CheckProposalConfig --config config.json +Quorum config --config config.json ``` *(Chains without proposals are automatically skipped.)* @@ -283,4 +283,5 @@ Contributions are welcome! Please open an issue or submit a pull request on [Git --- -**Happy Auditing!** If you have any questions or run into issues, please don’t hesitate to create a GitHub issue or open a discussion. \ No newline at end of file +**Happy Auditing!** If you have any questions or run into issues, please don’t hesitate to create a GitHub issue or open a discussion. +``` \ No newline at end of file diff --git a/version b/version index 73defc9..7adf240 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.171730.755013 +20250106.172642.134030 From b6445026bf244f3c00c7c562374fa099bfe8ecb9 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:26:59 +0200 Subject: [PATCH 20/24] . --- README.md | 1 - version | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 52ca59e..c428ceb 100644 --- a/README.md +++ b/README.md @@ -284,4 +284,3 @@ Contributions are welcome! Please open an issue or submit a pull request on [Git --- **Happy Auditing!** If you have any questions or run into issues, please don’t hesitate to create a GitHub issue or open a discussion. -``` \ No newline at end of file diff --git a/version b/version index 7adf240..fbcaab6 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.172642.134030 +20250106.172700.094680 From 9415e04ce5c0c22750ee71487ea9929658d5e6bf Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:28:21 +0200 Subject: [PATCH 21/24] . --- README.md | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c428ceb..2a1b30e 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ To fully enable Quorum’s checks, set the following: QUORUM_PATH="/path/to/quorum_artifacts" ``` -*(This file is automatically created by `SetupQuorum` if not already present.)* +*(This file is automatically created by `Quorum setup` if not already present.)* --- diff --git a/version b/version index fbcaab6..63af064 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.172700.094680 +20250106.172822.017833 From 69891e6fc576f9ff676b3f0b811d88d56e09f22a Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:50:28 +0200 Subject: [PATCH 22/24] Fix CI --- Quorum/checks/proposal_check.py | 1 - .../implementations/check_proposal_config.py | 34 ++++++++++++++++--- version | 2 +- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Quorum/checks/proposal_check.py b/Quorum/checks/proposal_check.py index 2436f36..ba5feec 100644 --- a/Quorum/checks/proposal_check.py +++ b/Quorum/checks/proposal_check.py @@ -10,7 +10,6 @@ import Quorum.utils.config_loader as ConfigLoader - class CustomerConfig(BaseModel): customer: str payload_addresses: list[PayloadAddresses] diff --git a/Quorum/entry_points/implementations/check_proposal_config.py b/Quorum/entry_points/implementations/check_proposal_config.py index 8d0fff8..5816c3c 100644 --- a/Quorum/entry_points/implementations/check_proposal_config.py +++ b/Quorum/entry_points/implementations/check_proposal_config.py @@ -1,7 +1,12 @@ import argparse -from typing import Dict, Any +from typing import Any -from Quorum.checks.proposal_check import run_customer_proposal_validation, ProposalConfig +from Quorum.checks.proposal_check import ( + run_customer_proposal_validation, + ProposalConfig, + CustomerConfig, + PayloadAddresses +) def run_config(args: argparse.Namespace) -> None: @@ -22,6 +27,27 @@ def run_config(args: argparse.Namespace) -> None: Raises: TypeError: If config_data cannot be unpacked into ProposalConfig """ - config_data: Dict[str, Any] = args.config - prop_config = ProposalConfig(**config_data) + config_data: dict[str, Any] = args.config + customers_config: list[CustomerConfig] = [] + + for customer_name, chains in config_data.items(): + payload_addresses: list[PayloadAddresses] = [] + + for chain_name, proposals in chains.items(): + proposal_addresses = proposals.get("Proposals", []) + payload_addresses.append( + PayloadAddresses( + chain=chain_name, + addresses=proposal_addresses + ) + ) + + customers_config.append( + CustomerConfig( + customer=customer_name, + payload_addresses=payload_addresses + ) + ) + + prop_config = ProposalConfig(customers_config=customers_config) run_customer_proposal_validation(prop_config) diff --git a/version b/version index 63af064..b49e831 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.172822.017833 +20250106.175028.966856 From 851b5748286f4915e654250b7b26329e774d1afe Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 17:56:17 +0200 Subject: [PATCH 23/24] . --- Quorum/entry_points/implementations/check_proposal_id.py | 9 +++++++-- version | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Quorum/entry_points/implementations/check_proposal_id.py b/Quorum/entry_points/implementations/check_proposal_id.py index a18814a..21d810b 100644 --- a/Quorum/entry_points/implementations/check_proposal_id.py +++ b/Quorum/entry_points/implementations/check_proposal_id.py @@ -1,7 +1,12 @@ import argparse from Quorum.apis.governance.aave_governance import AaveGovernanceAPI -from Quorum.checks.proposal_check import run_customer_proposal_validation, ProposalConfig +from Quorum.checks.proposal_check import ( + run_customer_proposal_validation, + ProposalConfig, + CustomerConfig, + PayloadAddresses +) CUSTOMER_TO_API = { @@ -34,6 +39,6 @@ def run_proposal_id(args: argparse.Namespace) -> None: api = CUSTOMER_TO_API[customer_key] payloads_addresses = api.get_all_payloads_addresses(proposal_id) - config = ProposalConfig(customers_config=[{"customer": customer, "payload_addresses": payloads_addresses}]) + config = ProposalConfig(customers_config=[CustomerConfig(customer=customer, payload_addresses=payloads_addresses)]) run_customer_proposal_validation(config) diff --git a/version b/version index b49e831..a7ba9b4 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.175028.966856 +20250106.175617.791589 From 551dc41849a0c1bd36d85d2a3711d6bb5fb565a7 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Mon, 6 Jan 2025 19:11:18 +0200 Subject: [PATCH 24/24] Improve description --- Quorum/entry_points/quorum_cli.py | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Quorum/entry_points/quorum_cli.py b/Quorum/entry_points/quorum_cli.py index c0f260b..10a0efd 100644 --- a/Quorum/entry_points/quorum_cli.py +++ b/Quorum/entry_points/quorum_cli.py @@ -93,7 +93,7 @@ def add_arguments(parser: argparse.ArgumentParser, arguments: list[cli_args.Argu def main(): parser = argparse.ArgumentParser( prog="Quorum", - description="Main Quorum CLI with multiple subcommands." + description="CLI tool for validating and analyzing blockchain governance proposals, including payload verification, IPFS content validation, and report generation." ) subparsers = parser.add_subparsers(dest="command", required=True) diff --git a/version b/version index a7ba9b4..121eb39 100644 --- a/version +++ b/version @@ -1 +1 @@ -20250106.175617.791589 +20250106.191119.109042