diff --git a/Manifest.toml b/Manifest.toml index b167bb3..70cb341 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -3,17 +3,38 @@ julia_version = "1.7.2" manifest_format = "2.0" +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings"] +git-tree-sha1 = "9310d9495c1eb2e4fa1955dd478660e2ecab1fbb" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.3" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.0" + [[deps.CommonSolve]] git-tree-sha1 = "68a0743f578349ada8bc911a5cbd5a2ef6ed6d1f" uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" version = "0.2.0" +[[deps.Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "96b0bc6c52df76506efc8a441c6cf1adcb1babc4" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.42.0" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" @@ -24,10 +45,61 @@ git-tree-sha1 = "f74e9d5388b8620b4cee35d4c5a618dd4dc547f4" uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" version = "1.3.0" +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.9.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "ae02104e835f219b8930c7664b8012c93475c340" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.3.2" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.11" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +[[deps.DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] +git-tree-sha1 = "129b104185df66e408edd6625d480b7f9e9823a0" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.18" + +[[deps.Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" @@ -55,12 +127,48 @@ git-tree-sha1 = "f550e6e32074c939295eb5ea6de31849ac2c9625" uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" version = "0.5.1" +[[deps.InlineStrings]] +deps = ["Parsers"] +git-tree-sha1 = "61feba885fac3a407465726d0c330b3055df897f" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.1.2" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.InvertedIndices]] +git-tree-sha1 = "bee5f1ef5bf65df56bdd2e40447590b272a5471f" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.1.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "StructTypes", "UUIDs"] git-tree-sha1 = "8c1f668b24d999fb47baf80436194fdccec65ad2" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" version = "1.9.4" +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -91,9 +199,18 @@ version = "1.0.3" deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.0.2" + [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -101,20 +218,50 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +[[deps.OrderedCollections]] +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + [[deps.Parsers]] deps = ["Dates"] git-tree-sha1 = "85b5da0fa43588c75bb1ff986493443f821c70b7" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.2.3" +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "28ef6c7ce353f0b35d0df0d5930e0d072c1f5b9b" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.1" + +[[deps.PrettyTables]] +deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] +git-tree-sha1 = "dfb54c4e414caa595a1f2ed759b160f5a3ddcba5" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "1.3.1" + [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + [[deps.Random]] deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + [[deps.Requires]] deps = ["UUIDs"] git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" @@ -130,6 +277,12 @@ version = "1.4.0" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "6a2f7d70512d205ca8c7ee31bfa9f142fe74310c" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.3.12" + [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -139,15 +292,63 @@ git-tree-sha1 = "38d88503f695eb0301479bc9b0d4320b378bafe5" uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" version = "0.8.2" +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.0.1" + +[[deps.SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + [[deps.StructTypes]] deps = ["Dates", "UUIDs"] git-tree-sha1 = "d24a825a95a6d98c385001212dc9020d609f2d4f" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" version = "1.8.1" +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "5ce79ce186cc678bbb5c5681ca3379d1ddae11a1" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.7.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.6" + [[deps.URIs]] git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" @@ -160,6 +361,24 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/Project.toml b/Project.toml index 5dadd00..b641551 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,8 @@ authors = ["The Graph Foundation"] version = "0.1.0" [deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" GraphQLClient = "09d831e3-9c21-47a9-bfd8-076871817219" Roots = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" diff --git a/src/AllocationOpt.jl b/src/AllocationOpt.jl index 1db6cfc..67dfa14 100644 --- a/src/AllocationOpt.jl +++ b/src/AllocationOpt.jl @@ -1,8 +1,29 @@ module AllocationOpt +export optimize_indexer_to_csv! + +using CSV +using DataFrames + include("exceptions.jl") include("graphrepository.jl") include("data.jl") include("optimize.jl") +function optimize_indexer_to_csv!(; + id::String, + whitelist::Union{Nothing,Vector{String}}, + blacklist::Union{Nothing,Vector{String}}, + csv_write_path::String, +) + url = "https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-mainnet" + repository = snapshot(; url=url, indexer_query=nothing, subgraph_query=nothing) + alloc, filtered = optimize(id, repository, whitelist, blacklist) + df = DataFrame( + "Subgraph ID" => collect(keys(alloc)), "Allocation in GRT" => collect(values(alloc)) + ) + df[!, "Subgraph Signal"] = map(x -> x.signal, filtered.subgraphs) + return CSV.write(csv_write_path, df) +end + end diff --git a/src/graphrepository.jl b/src/graphrepository.jl index 3f0975f..cc66872 100644 --- a/src/graphrepository.jl +++ b/src/graphrepository.jl @@ -64,7 +64,9 @@ function snapshot(; # Subgraphs if isnothing(subgraph_query) subgraph_query = GQLQuery( - Dict("orderBy" => "signalledTokens", "orderDirection" => "desc"), + Dict( + "first" => 1000, "orderBy" => "signalledTokens", "orderDirection" => "desc" + ), ["id", "signalledTokens"], ) end @@ -74,7 +76,10 @@ function snapshot(; # Indexers if isnothing(indexer_query) indexer_query = GQLQuery( - Dict("where" => Dict("stakedTokens_gte" => "100000000000000000000000")), + Dict( + "first" => 1000, + "where" => Dict("stakedTokens_gte" => "100000000000000000000000"), + ), [ "id", "delegatedTokens", diff --git a/src/optimize.jl b/src/optimize.jl index d659301..8352c6f 100644 --- a/src/optimize.jl +++ b/src/optimize.jl @@ -27,6 +27,7 @@ function optimize(optimize_id::String, repository::Repository, whitelist, blackl end whitelist_with_indexer = vcat(other_indexers, whitelist) + ids = map(x -> x.id, repository.indexers) indexer = first(filter(x -> x.id == optimize_id, repository.indexers)) σ = indexer.stake + indexer.delegation filtered_repo = whitelist_repo(whitelist_with_indexer, repository) @@ -39,5 +40,5 @@ function optimize(optimize_id::String, repository::Repository, whitelist, blackl # TODO: Hope's gas algorithm - return alloc + return alloc, filtered_repo end diff --git a/test/optimize-test.jl b/test/optimize-test.jl index 983d3e2..9cedc40 100644 --- a/test/optimize-test.jl +++ b/test/optimize-test.jl @@ -12,14 +12,15 @@ ) @test_throws ArgumentError optimize("0x000", fake_repository, ("0x010",), ("0x010",)) - allocations = optimize("0x000", fake_repository, nothing, nothing) + allocations, _ = optimize("0x000", fake_repository, nothing, nothing) + println(allocations) @test isapprox(allocations["0x010"], 4.2; atol=0.1) @test isapprox(allocations["0x011"], 0.8; atol=0.1) - allocations = optimize("0x000", fake_repository, nothing, ["0x010"]) + allocations, _ = optimize("0x000", fake_repository, nothing, ["0x010"]) @test allocations["0x011"] ≈ 5.0 - allocations = optimize("0x000", fake_repository, ["0x010"], nothing) + allocations, _ = optimize("0x000", fake_repository, ["0x010"], nothing) @test allocations["0x010"] ≈ 5.0 fake_repository = Repository( @@ -33,7 +34,7 @@ ], [Subgraph("0x010", 1.0), Subgraph("0x011", 5.0)], ) - allocations = optimize("0x000", fake_repository, nothing, nothing) + allocations, _ = optimize("0x000", fake_repository, nothing, nothing) @test allocations["0x010"] ≈ 0.0 @test allocations["0x011"] ≈ 2.0 end