redo plan generation, profit calculation, and creating actions

Project.toml

@@ -5,6 +5,7 @@ version = "1.2.0"
 
 [deps]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
 GraphQLClient = "09d831e3-9c21-47a9-bfd8-076871817219"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 Roots = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"

src/AllocationOpt.jl

@@ -1,5 +1,6 @@
 module AllocationOpt
 using DataFrames
+using Formatting
 
 export optimize_indexer
 
@@ -25,16 +26,14 @@ The optimizer will return a Julia DataFrame.
 """
 function optimize_indexer(;
     id::String,
-    grtgas::Float64,
-    alloc_lifetime::Int64,
     whitelist::Union{Nothing,Vector{String}},
     blacklist::Union{Nothing,Vector{String}},
 )
     url = "https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-mainnet"
     repository = snapshot(; url=url, indexer_query=nothing, subgraph_query=nothing)
     network = network_issuance(; url=url, network_id=nothing, network_query=nothing)
 
-    alloc, filtered = optimize(id, repository, grtgas, network, alloc_lifetime, whitelist, blacklist)
+    alloc, filtered = optimize(id, repository, whitelist, blacklist)
 
     df = DataFrame(
         "Subgraph ID" => collect(keys(alloc)), "Allocation in GRT" => collect(values(alloc))
@@ -62,46 +61,50 @@ function optimize_indexer(;
         optimize_indexer(; id, grtgas, alloc_lifetime, whitelist, blacklist)
     end
 
+    # Fetch data
     url = "https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-mainnet"
     repository = snapshot(; url=url, indexer_query=nothing, subgraph_query=nothing)
     network = network_issuance(; url=url, network_id=nothing, network_query=nothing)
     indexer::Indexer = repository.indexers[findfirst(x -> x.id == id, repository.indexers)]
 
+    # Do not consider any allocations younger than alloc_lifetime_threshold 
     young_list = filter_young_allocations(id, repository, alloc_lifetime_threshold, network)
-
     if isnothing(blacklist)
         blacklist = young_list
     else
-        append!(blacklist, young_list)
+        blacklist = append!(blacklist, young_list)
     end
 
-    alloc, filtered = optimize(id, repository, grtgas, network, alloc_lifetime, whitelist, blacklist)
+    # Optimize and create summary
+    alloc, filtered = optimize(id, repository, grtgas, network, alloc_lifetime, preference_threshold, whitelist, blacklist)
     alloc_list = filter(a -> a.amount != 0, map(alloc_id -> Allocation(alloc_id, alloc[alloc_id], network.current_epoch), collect(keys(alloc))))
-    actions = create_actions(id, filtered, repository, network, alloc_list, alloc_lifetime, grtgas, preference_threshold)
-
-    # if there are close actions, the optimized result should be updated - run optimizer again here as we put subgraphs to close into blacklist
-    blacklist = append!(map(action -> action.id, actions[1]), blacklist)
-
-    # run a second time for the updated blacklist
-    alloc, filtered = optimize(id, repository, grtgas, network, alloc_lifetime, whitelist, blacklist)
-    alloc_list = filter(a -> a.amount != 0, map(alloc_id -> Allocation(alloc_id, alloc[alloc_id], network.current_epoch), collect(keys(alloc))))
-    actions = create_actions(id, filtered, repository, network, alloc_list, alloc_lifetime, grtgas, preference_threshold)
-
-    # println("""- brief summary -
-    #         indexer: $(indexer.id) , available_stake: $(indexer.stake + indexer.delegation)
-    #         use gas in grt: $(grtgas) , use allocation lifetime: $(alloc_lifetime), number of allocations: $(length(filter(a -> a > 0.0, collect(values(alloc)))))
-    #         indexer_subgraph_rewards: $(sum(indexer_subgraph_rewards(filtered, network, alloc, alloc_lifetime)))
-    #         actions: $(actions)
-    #         """)
+    actions = create_actions(id, filtered, repository, network, alloc_list, alloc_lifetime, grtgas, preference_threshold, young_list)
 
     df = DataFrame(
         "Subgraph ID" => map(a -> a.id, alloc_list), "Allocation in GRT" => map(a -> a.amount, alloc_list)
     )
-    df[!, "Subgraph Signal"] = map(x -> x.signal, filtered.subgraphs)
-    df[!, "Subgraph Indexing Reward"] = subgraph_rewards(filtered, network, alloc_lifetime)
-    df[!, "Estimated Profit"] = estimated_profit(filtered, alloc_list, grtgas, network, alloc_lifetime)
-    # add rows for close actions. Currently only have Open or Reallocate
-    df[!, "Action"] = map(a -> a in actions[2] ? "Open" : "Reallocate" , alloc_list)
+    df[!, "Subgraph Signal"] = map(x -> x.signal, filter(sg -> sg.id in map(a->a.id,alloc_list), filtered.subgraphs))
+    df[!, "Indexing Reward"] = indexer_subgraph_rewards(filtered, network, alloc_list, alloc_lifetime)
+    # Add alloc rows for close actions? Currently only have allocations that want to Open or Reallocate
+    df[!, "Action"] = map(a -> a in actions[2] ? "Open" : (a in actions[3] ? "Reallocate" : "Do not open") , alloc_list)
+
+    # print_summary(indexer, df, alloc_list, actions[1], alloc_lifetime)
+
     return df
 end
+
+function print_summary(indexer, df, alloc_list, close_actions, alloc_lifetime)
+    println("""- brief summary -
+            indexer: $(indexer.id) , available_stake: $(indexer.stake + indexer.delegation)
+            use allocation lifetime: $(alloc_lifetime), number of allocations: $(length(alloc_list))
+            dataframe: $(df)
+            """)
+    for a in alloc_list
+        println("graph indexer rules set $(a.id) decisionBasis always allocationLifetime $(alloc_lifetime) allocationAmount $(format(a.amount))")
+    end
+    for a in close_actions
+        println("graph indexer rules stop ", a.id)
+    end
+end
+
 end

src/costbenefit.jl

@@ -44,6 +44,7 @@ function indicator_gas_fee(allocs::Vector{Allocation}, gas)
     calculate_gas_fee(allocs, gas) + calculate_gas_fee(allocs, gas) + calculate_gas_fee(allocs, claim_gas)
 end
 
+#TODO: prefer to use Vector{Allocation} over dictionaries, clean up format
 function estimated_profit(repo::Repository, allocs::Vector{Allocation}, gas::Float64, network::Network, alloc_lifetime::Int)
     return indexer_subgraph_rewards(repo, network, allocs, alloc_lifetime) - indicator_gas_fee(allocs, gas)
 end
@@ -52,7 +53,7 @@ function estimated_profit(repo::Repository, allocs::Dict{String, Float64}, gas::
     return indexer_subgraph_rewards(repo, network, allocs, alloc_lifetime) - indicator_gas_fee(allocs, gas)
 end
 
-# daily issuance
+# Daily issuance of the network
 function issued_token(network::Network)
     network.principle_supply * network.issuance_rate_per_block ^ network.block_per_epoch - network.principle_supply
 end
@@ -97,6 +98,7 @@ function does_exist(alloc_id::String, existing_allocations_in_plan)
     findfirst(y -> y.id == alloc_id, existing_allocations_in_plan)
 end
 
+# For existing allocations, take the rest of its lifetime and estimate rewards without gas
 function compare_rewards(indexer_id::String, filtered_repo::Repository, current_repo::Repository, network::Network, alloc_list::Vector{Allocation}, alloc_lifetime::Int, gas::Float64, preference_threshold::Float64)
     new_allocation_ids = map(a -> a.id, filter(a -> (a.id in map(a -> a.id, allocations_by_indexer(indexer_id, current_repo))), alloc_list))
 
@@ -108,30 +110,31 @@ function compare_rewards(indexer_id::String, filtered_repo::Repository, current_
                 a.id, 
                 allocation_amounts(indexer_id, current_repo, a.id).amount, 
                 (a.created_at_epoch + alloc_lifetime - network.current_epoch)
-            ))
+            ) - gas * 1.5)
     ), alloc_list)
 
     plan_estimated_profit = estimated_profit(filtered_repo, alloc_list, gas, network, alloc_lifetime)
 
     return plan_estimated_profit - reward_without_reallocate
 end
 
-function create_actions(indexer_id::String, filtered_repo::Repository, current_repo::Repository, network::Network, alloc_list::Vector{Allocation}, alloc_lifetime::Int, gas::Float64, preference_threshold::Float64)
+function create_actions(indexer_id::String, filtered_repo::Repository, current_repo::Repository, network::Network, alloc_list::Vector{Allocation}, alloc_lifetime::Int, gas::Float64, preference_threshold::Float64, young_allocation_ids::Union{Nothing,Vector{String}})
     # Identify allocations that need to be closed, reallocated, opened
     alloc_current = allocations_by_indexer(indexer_id, current_repo)
-    alloc_id_in_plan = map(a->a.id, alloc_list)
     alloc_id_current = map(a->a.id, alloc_current)
 
-    # Close the ones in indexer's current repo but not in alloc_list at all
-    close_actions::Vector{Allocation} = filter(a -> !(a.id in alloc_id_in_plan), alloc_current)
-    # Open the ones in indexer's alloc_list not in current repo at all
-    open_actions::Vector{Allocation} = filter(a -> !(a.id in alloc_id_current), alloc_list)
-
-    comparison = compare_rewards(indexer_id, filtered_repo, current_repo, network, alloc_list, alloc_lifetime, gas, preference_threshold)
-    indicators = allocation_indicator(comparison)
-
-    realloc_actions = map(i -> alloc_list[i], findall(isone, indicators))
-    close_actions = append!(map(i -> alloc_list[i], findall(iszero, indicators)), close_actions)
+    # Reallocate current allocations if compared rewards is positive
+    indicators = allocation_indicator(compare_rewards(indexer_id, filtered_repo, current_repo, network, alloc_list, alloc_lifetime, gas, preference_threshold))
+    worthy_allocations = map(i -> alloc_list[i], findall(isone, indicators))
+    worthy_allocation_ids = map(i -> i.id, worthy_allocations)
+    realloc_actions::Vector{Allocation} = filter(a -> a.id in alloc_id_current , worthy_allocations)
+    # Close current allocations in indexer's current repo that are not in alloc_list and make sure unworthy allocations cannot open, filtering out younger allocations
+    do_not_close_ids = isnothing(young_allocation_ids) ? worthy_allocation_ids : (young_allocation_ids ∪ worthy_allocation_ids)
+    close_actions::Vector{Allocation} = filter(a -> !(a.id in do_not_close_ids), alloc_current)
 
+    # Open the ones worthy not in current repo
+    open_actions::Vector{Allocation} = filter(a -> !(a.id in alloc_id_current) , worthy_allocations)
+
+    @assert length(close_actions ∩ open_actions ∩ realloc_actions) == 0
     return (close_actions, open_actions, realloc_actions)
 end

src/data.jl

@@ -93,6 +93,11 @@ function signal_shares(repo::Repository, network::Network, alloc_list::Vector{Al
     return map(x -> x.signal / total_signalled, filter(x-> x.id in ids, repo.subgraphs))
 end
 
+function stakes(repo::Repository, indexer_id::String)
+    indexer = repo.indexers[findfirst(x -> x.id == indexer_id, repo.indexers)]
+    return indexer.stake + indexer.delegation
+end
+
 function stakes(repo::Repository)
     return map(x -> x.stake + x.delegation, repo.indexers)
 end

src/graphrepository.jl

@@ -34,7 +34,7 @@ struct Indexer
             togrt(delegation),
             togrt(stake),
             map(
-                x -> Allocation(x["subgraphDeployment"]["id"], x["allocatedTokens"], x["createdAtEpoch"]),
+                x -> Allocation(x["subgraphDeployment"]["ipfsHash"], x["allocatedTokens"], x["createdAtEpoch"]),
                 allocation,
             ),
         )
@@ -82,13 +82,14 @@ function snapshot(;
     if isnothing(subgraph_query)
         subgraph_query = GQLQuery(
             Dict(
-                "first" => 1000, "orderBy" => "signalledTokens", "orderDirection" => "desc"
+                "first" => 1000, "orderBy" => "signalledTokens", "orderDirection" => "desc",
+                "where" => Dict("signalledTokens_gte" => "1000000000000000000000"),
             ),
-            ["id", "signalledTokens"],
+            ["ipfsHash", "signalledTokens"],
         )
     end
     subgraphs_data = query(client, "subgraphDeployments"; query_args=subgraph_query.args, output_fields=subgraph_query.fields).data["subgraphDeployments"]
-    subgraphs = map(x -> Subgraph(x["id"], x["signalledTokens"]), subgraphs_data)
+    subgraphs = map(x -> Subgraph(x["ipfsHash"], x["signalledTokens"]), subgraphs_data)
 
     # Indexers
     if isnothing(indexer_query)
@@ -101,7 +102,7 @@ function snapshot(;
                 "id",
                 "delegatedTokens",
                 "stakedTokens",
-                "allocations(where: {status:\"Active\"}){allocatedTokens,createdAtEpoch,subgraphDeployment{id}}",
+                "allocations(where: {status:\"Active\"}){allocatedTokens,createdAtEpoch,subgraphDeployment{ipfsHash}}",
             ],
         )
     end

src/optimize.jl

@@ -2,29 +2,38 @@ using Roots
 
 export optimize
 
-function optimize(optimize_id::String, repository::Repository, gas::Float64, network::Network, alloc_lifetime::Int, whitelist, blacklist)
+function optimize(optimize_id::String, repository::Repository, gas::Float64, network::Network, alloc_lifetime::Int64, preference_threshold::Float64, whitelist, blacklist)
     # Base case
     alloc, frepo = optimize(optimize_id, repository, whitelist, blacklist)
-    profit = sum(values(estimated_profit(frepo, alloc, gas, network, alloc_lifetime)))
-
-    # preset parameters
-    allocation_min_thresholds::Vector{Float64} = map(x -> 100000 + gas * x * 10000, 1:10)
 
+    # Create threshold based on suggested optimization without gas
+    allocation_min_thresholds::Vector{Float64} = sort!(unique(values(alloc)))
+
     # Filter whitelist, reducing the number of subgraphs
+    profit = 0.0
     for threshold in allocation_min_thresholds
         whitelist = filter(x -> alloc[x] > threshold, map(x -> x.id, frepo.subgraphs))
         if !isempty(whitelist)
             talloc, tfrepo = optimize(optimize_id, repository, whitelist, nothing)
             tprofit = sum(values(estimated_profit(tfrepo, talloc, gas, network, alloc_lifetime)))
             if tprofit >= profit
+                if tprofit > profit
+                end
                 alloc = talloc
                 frepo = tfrepo
                 profit = tprofit
             end
         end
     end
 
-    return alloc, frepo
+    # Create vector of allocations that should be created this epoch
+    alloc_list = filter(a -> a.amount != 0, map(alloc_id -> Allocation(alloc_id, alloc[alloc_id], network.current_epoch), collect(keys(alloc))))
+    actions = create_actions(optimize_id, frepo, repository, network, alloc_list, alloc_lifetime, gas, preference_threshold, blacklist)
+
+    # Excluding actions that are deemed to close, do final optimization
+    whitelist =  map(x -> x.id, actions[2] ∪ actions[3])
+
+    return optimize(optimize_id, repository, whitelist, nothing)
 end
 
 function optimize(optimize_id::String, repository::Repository, whitelist, blacklist)
@@ -64,7 +73,7 @@ function optimize(optimize_id::String, repository::Repository, whitelist, blackl
     alloc = Dict(sgraph_ids .=> ω)
 
     # Check the constraint as a test (+1 due to small rounding error
-    @assert (sum(values(alloc)) <= σ + 1)
+    @assert (sum(values(alloc)) >= σ - 1 && (sum(values(alloc)) <= σ + 1))
 
     return alloc, filtered_repo
 end

test/costbenefit-test.jl

@@ -42,7 +42,7 @@
     @test indexing_reward ≈ estimated_prof
 
     # After getting optimize to run correctly, check against trivial allocation above and compare_rewards
-    optimized_alloc_list, optimized_filterd_repo = optimize("1x001", fake_repository, gas, network, alloc_lifetime, nothing, nothing)
+    optimized_alloc_list, optimized_filterd_repo = optimize("1x001", fake_repository, gas, network, alloc_lifetime, preference, nothing, nothing)
     # add one for optimized_alloc_list = Dict("0x011" => 3.7132034355964256, "0x010" => 6.2867965644035735)
 
     # Whatever the original allocations are, the optimized solution should be at least as much
@@ -54,19 +54,17 @@
     @test sum(compared) >= 0
 
     # test create actions with different time in the network
-    actions = create_actions("1x001", optimized_filterd_repo, fake_repository, network, allocs_list, alloc_lifetime, gas, 1.0)
+    actions = create_actions("1x001", optimized_filterd_repo, fake_repository, network, allocs_list, alloc_lifetime, gas, 1.0, nothing)
     @test length(actions[3]) == 2
-    actions_2 = create_actions("1x001", optimized_filterd_repo, fake_repository, Network("1", 100.0, 1.0001, 15, 15.0, 0), allocs_list, alloc_lifetime, gas, 1.0)
-    @test length(actions_2[1]) == 2
-    actions_3 = create_actions("1x001", optimized_filterd_repo, fake_repository, Network("1", 100.0, 1.0001, 15, 15.0, 2), allocs_list, alloc_lifetime, gas, 1.0)
-    @test length(actions_3[1]) == 1 && length(actions_3[2]) == 0 && length(actions_3[3]) == 1
+    actions_2 = create_actions("1x001", optimized_filterd_repo, fake_repository, Network("1", 100.0, 1.0001, 15, 15.0, 0), allocs_list, alloc_lifetime, gas, 1.0, nothing)
+    @test length(actions_2[1]) == 1 && length(actions_2[3]) == 1
 
     # Test for the other player
-    optimized_alloc_list_2, optimized_filterd_repo_2 = optimize("1x000", fake_repository, gas, network, alloc_lifetime, nothing, nothing)
+    optimized_alloc_list_2, optimized_filterd_repo_2 = optimize("1x000", fake_repository, gas, network, alloc_lifetime, preference, nothing, nothing)
     alloc_list_2 = map(a -> Allocation(a, optimized_alloc_list_2[a], network.current_epoch), collect(keys(optimized_alloc_list_2)))
     compared = compare_rewards("1x000", optimized_filterd_repo_2, fake_repository, network, alloc_list_2, alloc_lifetime, gas, 1.0)
-    actions_2_1 = create_actions("1x000", optimized_filterd_repo_2, fake_repository, network, alloc_list_2, alloc_lifetime, gas, 1.0)
-    @test length(actions_2_1[1]) == 2
+    actions_3 = create_actions("1x000", optimized_filterd_repo_2, fake_repository, network, alloc_list_2, alloc_lifetime, gas, 1.0, nothing)
+    @test length(actions_3[1]) == 1
 
     # Test gas calculations
     gas = 0.1

test/optimize-test.jl

@@ -38,9 +38,9 @@
     @test allocs["0x011"] ≈ 2.0
 
     # Test incorporating gas fees, 
-    alloc_df = optimize_indexer(;id="0xc60d0c8c74b5d3a33ed51c007ebae682490de261", 
-                                grtgas=200.0, alloc_lifetime=14, whitelist=nothing, blacklist=nothing, alloc_lifetime_threshold=20, preference_threshold=1.00)
+    alloc_df = optimize_indexer(;id="0xa959b5afe73c6faa803541b5c4edc0492dfda294", 
+                                grtgas=500.0, alloc_lifetime=14, whitelist=nothing, blacklist=nothing, alloc_lifetime_threshold=20, preference_threshold=1.00)
 
     # println(alloc_df)
-    @test true
+    # @test true
 end