LLM tests + structure outputs

Quorum/ipfs_validator.py

@@ -72,10 +72,13 @@ def main():
     answer = ipfs_validation_chain.execute(
         prompt_templates = args.prompt_templates, ipfs=ipfs, payload=payload
     )
-
-    # Output the LLM's response
-    print(answer)
 
-
+    if answer.incompatibilities:
+        print("Found incompatibilities:")
+        for incompatibility in answer.incompatibilities:
+            print(incompatibility)
+    else:
+        print("LLM found no incompatibilities. Please Check manually.")
+
 if __name__ == '__main__':
     main()

Quorum/llm/chains/ipfs_validation_chain.py

@@ -1,11 +1,30 @@
-from Quorum.llm.jinja_utils import render_prompt
-from Quorum.llm.chains.cached_llm import CachedLLM
+from typing import Optional
+from pydantic import BaseModel, Field
 
 from langchain_core.messages import HumanMessage, SystemMessage
 from langgraph.checkpoint.memory import MemorySaver
 from langgraph.graph import START, MessagesState, StateGraph
-from langchain_core.output_parsers import StrOutputParser
 
+from Quorum.llm.jinja_utils import render_prompt
+from Quorum.llm.chains.cached_llm import CachedLLM
+
+class Incompatibility(BaseModel):
+    """
+    Incompatibility is a Pydantic model that represents a mismatch between the IPFS and Solidity payloads.
+    """
+    subject: str = Field(..., description="The subject of the incompatibility (e.g. disagreement between IPFS and Solidity).")
+    subject_in_ipfs: str = Field(..., description="The subject details as described in the IPFS payload.")
+    subject_in_solidity: str = Field(..., description="The subject details as described in the Solidity payload.")
+    description: str = Field(..., description="A detailed description of the incompatibility.")
+
+class IncompatibilityArray(BaseModel):
+    """
+    IncompatibilityArray is a Pydantic model that represents a list of incompatibilities between the IPFS and Solidity payloads.
+    """
+    incompatibilities: Optional[list[Incompatibility]] = Field(
+        ...,
+        description="A list of incompatibilities between the IPFS and Solidity payloads."
+    )
 
 class IPFSValidationChain(CachedLLM):
     """
@@ -24,6 +43,8 @@ def __init__(self):
         prompt templates for execution.
         """
         super().__init__()
+
+        self.structured_llm = self.llm.with_structured_output(IncompatibilityArray)
 
         # Define the workflow for the IPFS validation chain
         workflow = StateGraph(state_schema=MessagesState)
@@ -43,7 +64,7 @@ def __call_model(self, state: MessagesState) -> MessagesState:
         response = self.llm.invoke(messages)
         return {"messages": response}
 
-    def execute(self, prompt_templates: list[str], ipfs: str, payload: str, thread_id: int = 1) -> str:
+    def execute(self, prompt_templates: list[str], ipfs: str, payload: str, thread_id: int = 1) -> IncompatibilityArray:
         """
         Executes the IPFS validation workflow by rendering prompts, interacting with the LLM,
         and retrieving the final validation report.
@@ -74,4 +95,4 @@ def execute(self, prompt_templates: list[str], ipfs: str, payload: str, thread_i
                 config={"configurable": {"thread_id": f"{thread_id}"}},
             )
 
-        return StrOutputParser().parse(history["messages"][-1].content)
+        return self.structured_llm.invoke([h.content for h in history["messages"]])

Quorum/tests/conftest.py

@@ -40,3 +40,15 @@ def tmp_cache() -> Generator[Path, None, None]:
     cache.mkdir()
     yield cache
     shutil.rmtree(cache)
+
+
+@pytest.fixture
+def load_ipfs_and_code() -> tuple[str, str]:
+    llm_dir = RESOURCES_DIR / 'llm' / "ipfs_validation_chain"
+    ipfs_path = llm_dir / 'ipfs.txt'
+    source_code_path = llm_dir / 'source_code.sol'
+
+    ipfs_content = ipfs_path.read_text()
+    source_code = source_code_path.read_text()
+
+    return ipfs_content, source_code

Quorum/tests/expected/test_llm/first_deposit_chain.json

@@ -0,0 +1,39 @@
+{
+    "listings": [
+        {
+            "asset_symbol": "USDT",
+            "asset_address": "0xdAC17F958D2ee523a2206206994597C13D831ec7",
+            "supply_seed_amount": null,
+            "supply_indicator": false,
+            "approve_indicator": false
+        },
+        {
+            "asset_symbol": "WBTC",
+            "asset_address": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599",
+            "supply_seed_amount": null,
+            "supply_indicator": false,
+            "approve_indicator": false
+        },
+        {
+            "asset_symbol": "WETH",
+            "asset_address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            "supply_seed_amount": null,
+            "supply_indicator": false,
+            "approve_indicator": false
+        },
+        {
+            "asset_symbol": "YFI",
+            "asset_address": "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e",
+            "supply_seed_amount": null,
+            "supply_indicator": false,
+            "approve_indicator": false
+        },
+        {
+            "asset_symbol": "ZRX",
+            "asset_address": "0xE41d2489571d322189246DaFA5ebDe1F4699F498",
+            "supply_seed_amount": null,
+            "supply_indicator": false,
+            "approve_indicator": false
+        }
+    ]
+}

Quorum/tests/resources/llm/ipf_validation_chain/ipfs.txt

@@ -0,0 +1,74 @@
+Title: Reserve Factor Updates Late August
+Author: karpatkey_TokenLogic
+Discussions: https://governance.aave.com/t/arfc-increase-bridged-usdc-reserve-factor-across-all-deployments/17787
+Snapshot: https://snapshot.org/#/aave.eth/proposal/0x9cc7906f04f45cebeaa48a05ed281f49da00d89c4dd988a968272fa179f14d06
+
+Simple Summary
+
+This AIP shall implement the following parameter adjustments:
+
+- Increase Slope1 across Polygon v2 by 75bps;
+- Increase Reserve Factor (RF) on Ethereum v2 and Avalanche v2 by 5.00%;
+- Increase USDC.e and USDbC RF by 5.00% on Arbitrum, Optimism, Polygon, and Base.
+
+Motivation
+
+This AIP will reduce deposit yield for assets on Ethereum v2 and Avalanche v2 instances of Aave Protocol by increasing the RF by 5.00%. By increasing the RF a greater portion of the interest paid by borrowers is directed to the Aave DAO's treasury.
+
+This results in a lower deposit rate for users and encourages migration from v2 instances of the Aave Protocol to v3. User's funds are not at risk of liquidation and the borrowing rate remains unchanged.
+
+The RF across all USDC.e and USDbC reserves will be increased by 5.00% to encourage migration from bridged USDC to native USDC on each respective network.
+
+By increasing the Slope1 parameter by 75bps on Polygon v2, the cost of capital to users increases and further encourages migration to Polygon v3.
+
+Specification
+
+Slope1 Parameter 75bps Increases:
+
+Asset    | Market     | Current Slope1 | Proposed Slope1
+-------- | ---------- | -------------- | ---------------
+DAI      | Polygon v2 | 12.75%         | 13.50%
+USDT     | Polygon v2 | 12.75%         | 13.50%
+wBTC     | Polygon v2 | 7.75%          | 8.50%
+wETH     | Polygon v2 | 7.75%          | 8.50%
+USDC     | Polygon v2 | 12.75%         | 13.50%
+wMATIC   | Polygon v2 | 9.75%          | 10.50%
+
+Reserve Factor 5.00% Increases:
+
+Asset    | Market     | Current RF | Proposed RF
+-------- | ---------- | ---------- | -----------
+DAI.e    | Avalanche v2 | 65.00%   | 70.00%
+USDC.e   | Avalanche v2 | 65.00%   | 70.00%
+USDT.e   | Avalanche v2 | 65.00%   | 70.00%
+wAVAX    | Avalanche v2 | 65.00%   | 70.00%
+WBTC.e   | Avalanche v2 | 70.00%   | 75.00%
+WETH.e   | Avalanche v2 | 65.00%   | 70.00%
+DAI      | Ethereum v2  | 70.00%   | 75.00%
+LINK     | Ethereum v2  | 75.00%   | 80.00%
+USDC     | Ethereum v2  | 70.00%   | 75.00%
+USDT     | Ethereum v2  | 70.00%   | 75.00%
+wBTC     | Ethereum v2  | 75.00%   | 80.00%
+wETH     | Ethereum v2  | 70.00%   | 75.00%
+USDC.e   | Arbitrum     | 35.00%   | 40.00%
+USDC.e   | Optimism     | 35.00%   | 40.00%
+USDC.e   | Polygon      | 35.00%   | 40.00%
+USDbC    | Base         | 35.00%   | 40.00%
+USDC.e   | Gnosis       | 10.00%   | 15.00%
+
+References
+
+- Implementation:
+    - AaveV2Ethereum: https://github.com/bgd-labs/aave-proposals-v3/blob/.../AaveV2Ethereum_ReserveFactorUpdatesLateAugust_20240821.sol
+    - AaveV2Polygon: https://github.com/bgd-labs/aave-proposals-v3/blob/.../AaveV2Polygon_ReserveFactorUpdatesLateAugust_20240821.sol
+    - ... [additional links truncated for brevity]
+- Tests:
+    - AaveV2Ethereum: https://github.com/bgd-labs/aave-proposals-v3/blob/.../AaveV2Ethereum_ReserveFactorUpdatesLateAugust_20240821.t.sol
+    - ... [additional links truncated for brevity]
+- Discussions and Snapshots:
+    - USDCe updates: https://governance.aave.com/t/arfc-increase-bridged-usdc-reserve-factor-across-all-deployments/17787/7
+    - ... [additional links truncated for brevity]
+
+Copyright
+
+Copyright and related rights waived via CC0.

Quorum/tests/resources/llm/ipf_validation_chain/source_code.sol

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.0;
+
+import {IProposalGenericExecutor} from 'aave-helpers/src/interfaces/IProposalGenericExecutor.sol';
+import {AaveV2Ethereum, AaveV2EthereumAssets, ILendingPoolConfigurator} from 'aave-address-book/AaveV2Ethereum.sol';
+
+/**
+ * @title Reserve Factor Updates Late August
+ * @author karpatkey_TokenLogic
+ * - Snapshot: https://snapshot.org/#/aave.eth/proposal/0x9cc7906f04f45cebeaa48a05ed281f49da00d89c4dd988a968272fa179f14d06
+ * - Discussion: https://governance.aave.com/t/arfc-increase-bridged-usdc-reserve-factor-across-all-deployments/17787
+ */
+contract AaveV2Ethereum_ReserveFactorUpdatesLateAugust_20240821 is IProposalGenericExecutor {
+    ILendingPoolConfigurator public constant POOL_CONFIGURATOR =
+        ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR);
+
+    uint256 public constant DAI_RF = 75_00;
+    uint256 public constant LINK_RF = 80_00;
+    uint256 public constant USDC_RF = 75_00;
+    uint256 public constant USDT_RF = 75_00;
+    uint256 public constant WBTC_RF = 80_00;
+    uint256 public constant WETH_RF = 75_00;
+
+    function execute() external {
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.DAI_UNDERLYING, DAI_RF);
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.LINK_UNDERLYING, LINK_RF);
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDC_UNDERLYING, USDC_RF);
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDT_UNDERLYING, USDT_RF);
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WBTC_UNDERLYING, WBTC_RF);
+        POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WETH_UNDERLYING, WETH_RF);
+    }
+}

Quorum/tests/test_llm.py

@@ -0,0 +1,33 @@
+import pytest
+import json5 as json
+from pathlib import Path
+
+from Quorum.tests.conftest import load_ipfs_and_code
+import Quorum.tests.conftest as conftest
+from Quorum.llm.chains.ipfs_validation_chain import IPFSValidationChain
+from Quorum.llm.chains.first_deposit_chain import FirstDepositChain, ListingArray, ListingDetails
+from Quorum.apis.block_explorers.source_code import SourceCode
+
+def test_ipfs_validation_chain(load_ipfs_and_code):
+    ipfs_content, source_code = load_ipfs_and_code
+    chain = IPFSValidationChain()
+    prompt_templates = ['ipfs_validation_prompt_part1.j2', "ipfs_validation_prompt_part2.j2"]
+
+    result = chain.execute(prompt_templates, ipfs_content, source_code)
+
+    assert result is not None
+    assert isinstance(result.incompatibilities, list)
+    assert result.incompatibilities == []
+
+
+@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True)
+def test_first_deposit_chain(source_codes: list[SourceCode], tmp_output_path: Path):
+    chain = FirstDepositChain()
+    source_code = "\n".join(source_codes[1].file_content)
+
+    result = chain.execute(source_code)
+
+    expected_path = conftest.EXPECTED_DIR / 'test_llm' / 'first_deposit_chain.json'
+    with open(expected_path) as f:
+        expected = json.load(f)
+    assert ListingArray(**expected) == result