From e952f37876f4e6e70ab9485a19c6643f290f801f Mon Sep 17 00:00:00 2001 From: Alexander Lyon Date: Wed, 17 Apr 2024 12:45:08 +0100 Subject: [PATCH] create turbo-static for compile time graph analysis --- Cargo.lock | 227 +++++++++++--- Cargo.toml | 3 +- crates/turbo-static/Cargo.toml | 24 ++ crates/turbo-static/docker-compose.yml | 23 ++ crates/turbo-static/file/.fjall | Bin 0 -> 5 bytes .../journals/01k8_4n0f09_0f009n1b9c_082d/0 | Bin 0 -> 998 bytes .../journals/01k8_4n0f09_0f009n1b9c_082d/1 | 0 .../journals/01k8_4n0f09_0f009n1b9c_082d/2 | 0 .../journals/01k8_4n0f09_0f009n1b9c_082d/3 | 0 .../turbo-static/file/partitions/links/.lsm | Bin 0 -> 5 bytes .../file/partitions/links/config.json | 8 + .../file/partitions/links/levels.json | 9 + crates/turbo-static/here | Bin 0 -> 491 bytes crates/turbo-static/readme.md | 18 ++ crates/turbo-static/src/lsp_client.rs | 165 +++++++++++ crates/turbo-static/src/main.rs | 279 ++++++++++++++++++ crates/turbo-static/src/visitor.rs | 74 +++++ 17 files changed, 787 insertions(+), 43 deletions(-) create mode 100644 crates/turbo-static/Cargo.toml create mode 100644 crates/turbo-static/docker-compose.yml create mode 100644 crates/turbo-static/file/.fjall create mode 100644 crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/0 create mode 100644 crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/1 create mode 100644 crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/2 create mode 100644 crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/3 create mode 100644 crates/turbo-static/file/partitions/links/.lsm create mode 100644 crates/turbo-static/file/partitions/links/config.json create mode 100644 crates/turbo-static/file/partitions/links/levels.json create mode 100644 crates/turbo-static/here create mode 100644 crates/turbo-static/readme.md create mode 100644 crates/turbo-static/src/lsp_client.rs create mode 100644 crates/turbo-static/src/main.rs create mode 100644 crates/turbo-static/src/visitor.rs diff --git a/Cargo.lock b/Cargo.lock index 6a0d9cdd09f14f..bdee32648ed53b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1128,9 +1128,9 @@ checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -1344,9 +1344,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1354,7 +1354,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.1", + "windows-targets 0.52.5", ] [[package]] @@ -1905,9 +1905,9 @@ checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1981,11 +1981,10 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] @@ -2002,15 +2001,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] @@ -2024,14 +2019,21 @@ dependencies = [ ] [[package]] -name = "crossbeam-utils" -version = "0.8.16" +name = "crossbeam-skiplist" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" dependencies = [ - "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + [[package]] name = "crossterm" version = "0.25.0" @@ -2504,6 +2506,12 @@ dependencies = [ "nom", ] +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + [[package]] name = "downcast-rs" version = "1.2.0" @@ -2687,9 +2695,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fdeflate" @@ -2759,6 +2767,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fjall" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8b3cbbdfa775c311965846c523ae291327a5cc3e433479583922ff9527594" +dependencies = [ + "byteorder", + "crc32fast", + "fs_extra", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", +] + [[package]] name = "flate2" version = "1.0.28" @@ -3210,6 +3234,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "558b88954871f5e5b2af0e62e2e176c8bde7a6c2c4ed41b13d138d96da2e2cbd" +[[package]] +name = "guardian" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6817154789d2e9bb2af0486500e774af579d0e6539247044f06d803b141448b5" + [[package]] name = "h2" version = "0.3.24" @@ -4316,9 +4346,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" dependencies = [ "value-bag", ] @@ -4363,6 +4393,42 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "lsm-tree" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "792f0f9d75b518035f7247774580ba60ee211d381237946e1a40609433f5573f" +dependencies = [ + "byteorder", + "chrono", + "crc32fast", + "crossbeam-skiplist", + "double-ended-peekable", + "fs_extra", + "guardian", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rand 0.8.5", + "seahash", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "lsp-server" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248f65b78f6db5d8e1b1604b4098a28b43d21a8eb1deeca22b1c421b276c7095" +dependencies = [ + "crossbeam-channel", + "log", + "serde", + "serde_json", +] + [[package]] name = "lsp-types" version = "0.94.1" @@ -4376,6 +4442,28 @@ dependencies = [ "url", ] +[[package]] +name = "lsp-types" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e34d33a8e9b006cd3fc4fe69a921affa097bae4bb65f76271f4644f9a334365" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + [[package]] name = "mach" version = "0.3.2" @@ -5303,6 +5391,15 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + [[package]] name = "path-clean" version = "0.1.0" @@ -5315,6 +5412,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "path-slash" version = "0.2.1" @@ -5999,6 +6105,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick_cache" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "347e1a588d1de074eeb3c00eadff93db4db65aeb62aee852b1efd0949fe65b6c" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -6849,9 +6965,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" dependencies = [ "serde_derive", ] @@ -6888,9 +7004,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" dependencies = [ "proc-macro2", "quote", @@ -6910,9 +7026,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "indexmap 2.2.3", "itoa", @@ -6922,10 +7038,11 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ + "itoa", "serde", ] @@ -7445,6 +7562,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + [[package]] name = "stop-token" version = "0.7.0" @@ -9181,15 +9304,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.3.5", + "fastrand 2.1.0", "rustix 0.38.31", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -9781,7 +9903,7 @@ dependencies = [ "dashmap", "futures", "httparse", - "lsp-types", + "lsp-types 0.94.1", "memchr", "serde", "serde_json", @@ -10044,6 +10166,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "turbo-static" +version = "0.1.0" +dependencies = [ + "bincode", + "clap 4.5.2", + "crossbeam-channel", + "fjall", + "ignore", + "itertools 0.10.5", + "lsp-server", + "lsp-types 0.95.1", + "proc-macro2", + "serde_json", + "serde_path_to_error", + "syn 2.0.58", + "tracing", + "tracing-subscriber", + "walkdir", +] + [[package]] name = "turbo-tasks" version = "0.1.0" @@ -11707,9 +11850,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" +checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" [[package]] name = "vcpkg" @@ -11968,9 +12111,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", diff --git a/Cargo.toml b/Cargo.toml index a03b0b7aed82aa..01634ba05709a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,8 @@ resolver = "2" members = [ "crates/node-file-trace", - "crates/tower-uds", + "crates/tower-uds", + "crates/turbo-static", "crates/turbo-tasks*", "crates/turbopack*", "crates/turborepo*", diff --git a/crates/turbo-static/Cargo.toml b/crates/turbo-static/Cargo.toml new file mode 100644 index 00000000000000..a17c1dab416fb1 --- /dev/null +++ b/crates/turbo-static/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "turbo-static" +version = "0.1.0" +edition = "2021" + +[dependencies] +bincode = "1.3.3" +clap = { workspace = true, features = ["derive"] } +crossbeam-channel = "0.5.12" +fjall = { version = "0.6.3", features = ["bloom"] } +ignore = "0.4.22" +itertools.workspace = true +lsp-server = "0.7.6" +lsp-types = "0.95.1" +proc-macro2 = { workspace = true, features = ["span-locations"] } +serde_json.workspace = true +serde_path_to_error = "0.1.16" +syn = { version = "2", features = ["parsing", "full", "visit", "extra-traits"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tracing.workspace = true +walkdir = "2.5.0" + +[lints] +workspace = true diff --git a/crates/turbo-static/docker-compose.yml b/crates/turbo-static/docker-compose.yml new file mode 100644 index 00000000000000..1ef424a6a866ad --- /dev/null +++ b/crates/turbo-static/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3" + +services: + memgraph: + image: memgraph/memgraph-mage:latest + container_name: memgraph-1 + pull_policy: always + ports: + - "7687:7687" + - "7444:7444" + command: ["--log-level=TRACE"] + + lab: + image: memgraph/lab:latest + container_name: memgraph-2 + pull_policy: always + ports: + - "3000:3000" + depends_on: + - memgraph + environment: + - QUICK_CONNECT_MG_HOST=memgraph + - QUICK_CONNECT_MG_PORT=7687 diff --git a/crates/turbo-static/file/.fjall b/crates/turbo-static/file/.fjall new file mode 100644 index 0000000000000000000000000000000000000000..9f91fa72031d4c535a95d94869972e59f71aa68e GIT binary patch literal 5 McmZ?s@?l^A00VCT-2eap literal 0 HcmV?d00001 diff --git a/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/0 b/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/0 new file mode 100644 index 0000000000000000000000000000000000000000..6f246325230736c6774e170c5cc49150e90462db GIT binary patch literal 998 zcmcIjyGjH>5S(2x5Hv7R%!D&t2kzv+O-xKQQuM&=+(&y1H#;LeyZ1m)L-7*?{Sd#w zz)$cKOwH`xyOW`eHj>KP$~@CapjVp%IQbVA9V19UP%J+(%jpdDb8Wy%bs zfk}QHB}17P7+aZOl!L1dd>tfO1x2p3lMWM!g~?S0eK0vrchR*FoYkY*McCdGceQ$d zX5va|2&G(oV-{m(G5=<(^IWg-8!5CruK)l5 literal 0 HcmV?d00001 diff --git a/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/1 b/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/1 new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/2 b/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/2 new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/3 b/crates/turbo-static/file/journals/01k8_4n0f09_0f009n1b9c_082d/3 new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/crates/turbo-static/file/partitions/links/.lsm b/crates/turbo-static/file/partitions/links/.lsm new file mode 100644 index 0000000000000000000000000000000000000000..e3f824d0ebeca22e0d817a2083dfb5871177c40b GIT binary patch literal 5 McmeYX_GMrI00XoD?EnA( literal 0 HcmV?d00001 diff --git a/crates/turbo-static/file/partitions/links/config.json b/crates/turbo-static/file/partitions/links/config.json new file mode 100644 index 00000000000000..bd48eae4b0f87e --- /dev/null +++ b/crates/turbo-static/file/partitions/links/config.json @@ -0,0 +1,8 @@ +{ + "path": "/Users/arlyon/Programming/nextpack/turbo/crates/turbo-static/file/partitions/links", + "block_size": 4096, + "level_count": 7, + "level_ratio": 8, + "type": "Standard", + "compression": "Lz4" +} \ No newline at end of file diff --git a/crates/turbo-static/file/partitions/links/levels.json b/crates/turbo-static/file/partitions/links/levels.json new file mode 100644 index 00000000000000..8dccb26c596e7b --- /dev/null +++ b/crates/turbo-static/file/partitions/links/levels.json @@ -0,0 +1,9 @@ +[ + [], + [], + [], + [], + [], + [], + [] +] \ No newline at end of file diff --git a/crates/turbo-static/here b/crates/turbo-static/here new file mode 100644 index 0000000000000000000000000000000000000000..6632d9d5a5a281ce075cd9f34a90428a058de21d GIT binary patch literal 491 zcmcJL!3x4K5JanY5B`Cm*Sa7g*s~wtLC-afYY9yg*-fhbdy_UEyxWCg*ttAr=Gk~E zIb0cuL6GjPfiu~fMB6G;gTc3Z3fX}3R9TQD9P;?YIG0-G-YLz%B!Rqfs>KqGqkc=g zCS>, + receiver: Option>, +} + +impl RAClient { + /// Create a new LSP client for Rust Analyzer. + pub fn new() -> Self { + let stdin = process::Stdio::piped(); + let stdout = process::Stdio::piped(); + let stderr = process::Stdio::inherit(); + + let child = process::Command::new("rust-analyzer") + .stdin(stdin) + .stdout(stdout) + .stderr(stderr) + // .env("RA_LOG", "info") + .spawn() + .expect("Failed to start RA LSP server"); + Self { + handle: child, + sender: None, + receiver: None, + } + } + + pub fn start(&mut self, folders: &[PathBuf]) { + let mut stdout = self.handle.stdout.take().unwrap(); + let mut stdin = self.handle.stdin.take().unwrap(); + + let (writer_sender, writer_receiver) = bounded::(0); + let writer = std::thread::spawn(move || { + writer_receiver + .into_iter() + .try_for_each(|it| it.write(&mut stdin)) + }); + + let (reader_sender, reader_receiver) = bounded::(0); + let reader = std::thread::spawn(move || { + let mut reader = std::io::BufReader::new(stdout); + while let Ok(Some(msg)) = Message::read(&mut reader) { + reader_sender + .send(msg) + .expect("receiver was dropped, failed to send a message"); + } + }); + + self.sender = Some(writer_sender); + self.receiver = Some(reader_receiver); + + let root_path = std::fs::canonicalize(folders.first().unwrap()) + .unwrap() + .to_string_lossy() + .to_string(); + let workspace_paths = folders + .iter() + .map(|p| std::fs::canonicalize(p).unwrap()) + .map(|p| lsp_types::WorkspaceFolder { + name: p.file_name().unwrap().to_string_lossy().to_string(), + uri: lsp_types::Url::from_file_path(p).unwrap(), + }) + .collect::>(); + + let resp = self.request(lsp_server::Request { + id: 1.into(), + method: "initialize".to_string(), + params: serde_json::to_value(&lsp_types::InitializeParams { + root_uri: Some(lsp_types::Url::from_file_path(&root_path).unwrap()), + root_path: Some(root_path), + workspace_folders: Some(workspace_paths), + process_id: Some(std::process::id()), + capabilities: lsp_types::ClientCapabilities { + workspace: Some(lsp_types::WorkspaceClientCapabilities { + workspace_folders: Some(true), + ..Default::default() + }), + ..Default::default() + }, + initialization_options: None, + work_done_progress_params: lsp_types::WorkDoneProgressParams { + work_done_token: Some(lsp_types::ProgressToken::String("prepare".to_string())), + }, + trace: None, + client_info: None, + locale: None, + }) + .unwrap(), + }); + + let resp = self.notify(lsp_server::Notification { + method: "initialized".to_string(), + params: serde_json::to_value(&lsp_types::InitializedParams {}).unwrap(), + }); + } + + pub fn request(&mut self, message: lsp_server::Request) -> lsp_server::Response { + tracing::debug!("sending {:?}", message); + self.sender + .as_mut() + .unwrap() + .send(Message::Request(message)) + .expect("failed to send message"); + + loop { + match self + .receiver + .as_mut() + .unwrap() + .recv() + .expect("failed to receive message") + { + lsp_server::Message::Response(response) => { + tracing::debug!("received {:?}", response); + return response; + } + m => tracing::trace!("unexpected message: {:?}", m), + } + } + } + + pub fn notify(&mut self, message: lsp_server::Notification) { + self.sender + .as_mut() + .unwrap() + .send(Message::Notification(message)) + .expect("failed to send message"); + } +} + +impl Drop for RAClient { + fn drop(&mut self) { + if self.sender.is_some() { + let resp = self.request(lsp_server::Request { + id: 1.into(), + method: "shutdown".to_string(), + params: serde_json::to_value(&()).unwrap(), + }); + + if resp.error.is_none() { + tracing::info!("shutting down RA LSP server"); + self.notify(lsp_server::Notification { + method: "exit".to_string(), + params: serde_json::to_value(&()).unwrap(), + }); + self.handle + .wait() + .expect("failed to wait for RA LSP server"); + tracing::info!("shut down RA LSP server"); + } else { + tracing::error!("failed to shutdown RA LSP server: {:#?}", resp); + } + } + + self.sender = None; + self.receiver = None; + } +} diff --git a/crates/turbo-static/src/main.rs b/crates/turbo-static/src/main.rs new file mode 100644 index 00000000000000..965647e5109ee2 --- /dev/null +++ b/crates/turbo-static/src/main.rs @@ -0,0 +1,279 @@ +use std::{collections::HashMap, error::Error, fs, path::PathBuf}; + +use clap::Parser; +use fjall::{Config, Keyspace, PartitionCreateOptions}; +use itertools::Itertools; +use lsp_server::{Connection, Message, Request, RequestId, Response}; +use lsp_types::ClientCapabilities; + +mod lsp_client; +mod visitor; + +#[derive(Parser)] +struct Opt { + #[clap(required = true)] + paths: Vec, + + /// reparse all files + #[clap(long)] + reparse: bool, + + /// reindex all files + #[clap(long)] + reindex: bool, +} + +fn main() -> Result<(), Box> { + tracing_subscriber::fmt::init(); + let opt = Opt::parse(); + + let mut connection = lsp_client::RAClient::new(); + connection.start(&opt.paths); + + // Each partition is its own physical LSM-tree + let fjall = Config::new("file").open()?; + + tracing::info!("getting tasks"); + let tasks = get_all_tasks(&opt.paths); + let dep_tree = resolve_tasks(&tasks, &mut connection, &fjall, opt.reindex); + let concurrency = resolve_concurrency(&dep_tree); + + write_dep_tree(&dep_tree, std::path::Path::new("here")); + + Ok(()) +} + +/// search the given folders recursively and attempt to find all tasks inside +#[tracing::instrument(skip_all)] +fn get_all_tasks(folders: &[PathBuf]) -> Vec<(PathBuf, (syn::Ident, Vec))> { + let mut out = vec![]; + + for folder in folders { + let walker = ignore::Walk::new(folder); + for entry in walker { + let entry = entry.unwrap(); + let rs_file = if let Some(true) = entry.file_type().map(|t| t.is_file()) { + let path = entry.path(); + let ext = path.extension().unwrap_or_default(); + if ext == "rs" { + std::fs::canonicalize(path).unwrap() + } else { + continue; + } + } else { + continue; + }; + + let file = fs::read_to_string(&rs_file).unwrap(); + let lines = file.lines(); + let mut occurences = vec![]; + + tracing::debug!("processing {}", rs_file.display()); + + for ((_, line), (line_no, fn_line)) in lines.enumerate().tuple_windows() { + if line.contains("turbo_tasks::function") { + tracing::debug!("found at {:?}:L{}", rs_file, line_no); + let task_name = fn_line.to_owned(); + occurences.push(line_no + 1); + } + } + + if occurences.is_empty() { + continue; + } + + // parse the file using syn and get the span of the functions + let file = syn::parse_file(&file).unwrap(); + let occurences_count = occurences.len(); + let mut visitor = visitor::TaskVisitor::new(); + syn::visit::visit_file(&mut visitor, &file); + if visitor.results.len() != occurences_count { + tracing::warn!( + "file {:?} passed the heuristic with {:?} but the visitor found {:?}", + rs_file, + occurences_count, + visitor.results.len() + ); + } + + out.extend( + visitor + .results + .into_iter() + .map(move |ident| (rs_file.clone(), ident)), + ) + } + } + + out +} + +fn resolve_tasks( + tasks: &[(PathBuf, (syn::Ident, Vec))], + client: &mut lsp_client::RAClient, + fjall: &fjall::Keyspace, + reindex: bool, +) -> HashMap> { + let items = fjall + .open_partition("links", PartitionCreateOptions::default()) + .unwrap(); + + let items = if reindex { + fjall.delete_partition(items).unwrap(); + fjall + .open_partition("links", PartitionCreateOptions::default()) + .unwrap() + } else { + items + }; + + tracing::info!( + "found {} tasks, of which {} cached", + tasks.len(), + items.len().unwrap() + ); + + let mut out = HashMap::new(); + + for (path, (ident, tags)) in tasks { + let key = format!( + "{}#{}:{}", + path.display(), + ident.to_string(), + ident.span().start().line + ); + if let Some(data) = items.get(&key).unwrap() { + tracing::info!("skipping {}: got data {:?}", key, data); + + let data: Vec<(String, lsp_types::Range)> = bincode::deserialize(&data).unwrap(); + out.insert(key, data); + continue; + }; + + tracing::info!("checking {} in {}", ident, path.display()); + + let mut count = 0; + let response = loop { + let response = client.request(lsp_server::Request { + id: 1.into(), + method: "textDocument/prepareCallHierarchy".to_string(), + params: serde_json::to_value(&lsp_types::CallHierarchyPrepareParams { + text_document_position_params: lsp_types::TextDocumentPositionParams { + position: lsp_types::Position { + line: ident.span().start().line as u32 - 1, // 0-indexed + character: ident.span().start().column as u32, + }, + text_document: lsp_types::TextDocumentIdentifier { + uri: lsp_types::Url::from_file_path(&path).unwrap(), + }, + }, + work_done_progress_params: lsp_types::WorkDoneProgressParams { + work_done_token: Some(lsp_types::ProgressToken::String( + "prepare".to_string(), + )), + }, + }) + .unwrap(), + }); + if let Some(Some(value)) = response.result.as_ref().map(|r| r.as_array()) { + if value.len() != 0 { + break value.to_owned(); + } + count += 1; + } + + // textDocument/prepareCallHierarchy will sometimes return an empty array so try + // at most 5 times + if count > 5 { + tracing::warn!("discovered isolated task {} in {}", ident, path.display()); + break vec![]; + } + + std::thread::sleep(std::time::Duration::from_secs(1)); + }; + + // callHierarchy/incomingCalls + let response = client.request(lsp_server::Request { + id: 1.into(), + method: "callHierarchy/incomingCalls".to_string(), + params: serde_json::to_value(&lsp_types::CallHierarchyIncomingCallsParams { + partial_result_params: lsp_types::PartialResultParams::default(), + item: lsp_types::CallHierarchyItem { + name: ident.to_string().clone(), + kind: lsp_types::SymbolKind::FUNCTION, + data: None, + tags: None, + detail: None, + uri: lsp_types::Url::from_file_path(&path).unwrap(), + range: lsp_types::Range { + start: lsp_types::Position { + line: ident.span().start().line as u32 - 1, // 0-indexed + character: ident.span().start().column as u32, + }, + end: lsp_types::Position { + line: ident.span().end().line as u32, + character: ident.span().end().column as u32, + }, + }, + selection_range: lsp_types::Range { + start: lsp_types::Position { + line: ident.span().start().line as u32 - 1, // 0-indexed + character: ident.span().start().column as u32, + }, + end: lsp_types::Position { + line: ident.span().end().line as u32 - 1, // 0-indexed + character: ident.span().end().column as u32, + }, + }, + }, + work_done_progress_params: lsp_types::WorkDoneProgressParams { + work_done_token: Some(lsp_types::ProgressToken::String("prepare".to_string())), + }, + }) + .unwrap(), + }); + + let response: Result, _> = + serde_path_to_error::deserialize(response.result.unwrap()); + + let links = response + .unwrap() + .into_iter() + .map(|i| (i.from.uri.to_string(), i.from.selection_range)) + .collect::>(); + + let data = bincode::serialize(&links).unwrap(); + + tracing::info!("links: {:?}: {:?}", links, data); + + items.insert(key, data).unwrap(); + fjall.persist().unwrap(); + return out; + } + + out +} + +enum CallingStyle { + Once, + ZeroOrOnce, + ZeroOrMore, + OneOrMore, +} + +/// given a map of tasks and functions that call it, produce a map of tasks and +/// those tasks that it calls + +fn resolve_concurrency( + dep_tree: &HashMap>, +) -> HashMap> { + Default::default() +} + +fn write_dep_tree( + dep_tree: &HashMap>, + out: &std::path::Path, +) { + let mut out = std::fs::File::create(out).unwrap(); + bincode::serialize_into(&mut out, dep_tree).unwrap(); +} diff --git a/crates/turbo-static/src/visitor.rs b/crates/turbo-static/src/visitor.rs new file mode 100644 index 00000000000000..354951463b1cd4 --- /dev/null +++ b/crates/turbo-static/src/visitor.rs @@ -0,0 +1,74 @@ +//! A visitor that traverses the AST and collects all functions or methods that +//! are annotated with `#[turbo_tasks::function]`. + +use std::collections::HashMap; + +use syn::{visit::Visit, Meta}; + +pub struct TaskVisitor { + pub results: Vec<(syn::Ident, Vec)>, +} + +impl TaskVisitor { + pub fn new() -> Self { + Self { + results: Default::default(), + } + } +} + +impl Visit<'_> for TaskVisitor { + #[tracing::instrument(skip_all)] + fn visit_item_fn(&mut self, i: &syn::ItemFn) { + if let Some(tags) = extract_tags(i.attrs.iter()) { + tracing::trace!("L{}: {}", i.sig.ident.span().start().line, i.sig.ident,); + self.results.push((i.sig.ident.clone(), tags)); + } + } + + #[tracing::instrument(skip_all)] + fn visit_impl_item_fn(&mut self, i: &syn::ImplItemFn) { + if let Some(tags) = extract_tags(i.attrs.iter()) { + tracing::trace!("L{}: {}", i.sig.ident.span().start().line, i.sig.ident,); + self.results.push((i.sig.ident.clone(), tags)); + } + } +} + +fn extract_tags<'a>(mut meta: impl Iterator) -> Option> { + meta.find_map(|a| match &a.meta { + // path has two segments, turbo_tasks and function + Meta::Path(path) if path.segments.len() == 2 => { + let first = &path.segments[0]; + let second = &path.segments[1]; + (first.ident == "turbo_tasks" && second.ident == "function").then(|| vec![]) + } + Meta::List(list) if list.path.segments.len() == 2 => { + let first = &list.path.segments[0]; + let second = &list.path.segments[1]; + if (first.ident != "turbo_tasks" || second.ident != "function") { + return None; + } + + // collect ident tokens as args + let tags: Vec<_> = list + .tokens + .clone() + .into_iter() + .filter_map(|t| { + if let proc_macro2::TokenTree::Ident(ident) = t { + Some(ident.to_string()) + } else { + None + } + }) + .collect(); + + Some(tags) + } + _ => { + tracing::trace!("skipping unknown annotation"); + None + } + }) +}