new repo for custom mvr indexer

crates/mvr-indexer/Cargo.toml

@@ -0,0 +1,89 @@
+[package]
+name = "sui-indexer"
+version.workspace = true
+authors = ["Mysten Labs <[email protected]>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+anyhow.workspace = true
+rand = "0.8.5"
+async-trait.workspace = true
+axum.workspace = true
+backoff.workspace = true
+bb8 = "0.8.5"
+bcs.workspace = true
+bytes.workspace = true
+chrono.workspace = true
+clap = { workspace = true, features = ["env"] }
+csv.workspace = true
+diesel = { workspace = true, features = ["chrono", "serde_json"] }
+diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] }
+futures.workspace = true
+hex.workspace = true
+indicatif.workspace = true
+itertools.workspace = true
+jsonrpsee.workspace = true
+object_store.workspace = true
+prometheus.workspace = true
+rayon.workspace = true
+regex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_with.workspace = true
+strum.workspace = true
+strum_macros.workspace = true
+tap.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["full"] }
+tokio-util = { workspace = true, features = ["rt"] }
+toml.workspace = true
+tracing.workspace = true
+url.workspace = true
+
+fastcrypto = { workspace = true, features = ["copy_key"] }
+mysten-metrics.workspace = true
+simulacrum.workspace = true
+sui-config.workspace = true
+sui-archival.workspace = true
+sui-core.workspace = true
+sui-data-ingestion-core.workspace = true
+sui-json.workspace = true
+sui-json-rpc.workspace = true
+sui-json-rpc-api.workspace = true
+sui-json-rpc-types.workspace = true
+sui-open-rpc.workspace = true
+sui-sdk.workspace = true
+sui-snapshot.workspace = true
+sui-storage.workspace = true
+sui-types.workspace = true
+sui-package-resolver.workspace = true
+sui-protocol-config.workspace = true
+telemetry-subscribers.workspace = true
+sui-rest-api.workspace = true
+sui-transaction-builder.workspace = true
+sui-synthetic-ingestion.workspace = true
+
+move-core-types.workspace = true
+move-bytecode-utils.workspace = true
+move-binary-format.workspace = true
+
+diesel_migrations.workspace = true
+cached.workspace = true
+tokio-stream.workspace = true
+dashmap.workspace = true
+
+[dev-dependencies]
+sui-keys.workspace = true
+sui-move-build.workspace = true
+sui-swarm-config.workspace = true
+sui-test-transaction-builder.workspace = true
+test-cluster.workspace = true
+ntest.workspace = true
+criterion.workspace = true
+
+[[bin]]
+name = "sui-indexer"
+path = "src/main.rs"

crates/mvr-indexer/README.md

@@ -0,0 +1,133 @@
+Sui indexer is an off-fullnode service to serve data from Sui protocol, including both data directly generated from chain and derivative data.
+
+⚠ **Warning:** Sui indexer is still experimental and we expect occasional breaking changes that require backfills.
+
+## Architecture
+![enhanced_FN](https://user-images.githubusercontent.com/106119108/221022505-a1d873c6-60e2-45f1-b2aa-e50192c4dfbb.png)
+
+## Steps to run locally
+### Prerequisites
+- install local [Postgres server](https://www.postgresql.org/download/). You can also `brew install postgresql@15` and then add the following to your `~/.zshrc` or `~/.zprofile`, etc:
+```sh
+export LDFLAGS="-L/opt/homebrew/opt/postgresql@15/lib"
+export CPPFLAGS="-I/opt/homebrew/opt/postgresql@15/include"
+export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
+```
+- make sure you have libpq installed: `brew install libpq`, and in your profile, add `export PATH="/opt/homebrew/opt/libpq/bin:$PATH"`. If this doesn't work, try `brew link --force libpq`.
+
+- install Diesel CLI with `cargo install diesel_cli --no-default-features --features postgres`, refer to [Diesel Getting Started guide](https://diesel.rs/guides/getting-started) for more details
+- [optional but handy] Postgres client like [Postico](https://eggerapps.at/postico2/), for local check, query execution etc.
+
+### Start the Postgres Service
+
+Postgres must run as a service in the background for other tools to communicate with.  If it was installed using homebrew, it can be started as a service with:
+
+``` sh
+brew services start postgresql@version
+```
+
+### Local Development(Recommended)
+
+See the [docs](https://docs.sui.io/guides/developer/getting-started/local-network) for detailed information. Below is a quick start guide:
+
+Start a local network using the `sui` binary:
+```sh
+cargo run --bin sui -- start --with-faucet --force-regenesis
+```
+
+If you want to run a local network with the indexer enabled (note that `libpq` is required), you can run the following command after following the steps in the next section to set up an indexer DB:
+```sh
+cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2
+```
+
+### Running standalone indexer
+1. DB setup, under `sui/crates/sui-indexer` run:
+```sh
+# an example DATABASE_URL is "postgres://postgres:postgres@localhost/exampledb"
+diesel setup --database-url="<DATABASE_URL>"
+diesel database reset --database-url="<DATABASE_URL>"
+```
+Note that you need an existing database for this to work. Using the DATABASE_URL example in the comment of the previous code, replace `exampledb` with the name of your database.
+
+2. Checkout to your target branch
+
+For example, if you want to be on the DevNet branch
+```sh
+git fetch upstream devnet && git reset --hard upstream/devnet
+```
+3. Start indexer binary, under `sui/crates/sui-indexer` run:
+- run indexer as a writer, which pulls data from fullnode and writes data to DB
+```sh
+# Change the RPC_CLIENT_URL to http://0.0.0.0:9000 to run indexer against local validator & fullnode
+cargo run --bin sui-indexer -- --db-url "<DATABASE_URL>" --rpc-client-url "https://fullnode.devnet.sui.io:443" --fullnode-sync-worker --reset-db
+```
+- run indexer as a reader, which is a JSON RPC server with the [interface](https://docs.sui.io/sui-api-ref#suix_getallbalances)
+```
+cargo run --bin sui-indexer -- --db-url "<DATABASE_URL>" --rpc-client-url "https://fullnode.devnet.sui.io:443" --rpc-server-worker
+```
+More flags info can be found in this [file](src/main.rs#L41).
+
+### DB reset
+When making db-related changes, you may find yourself having to run migrations and reset dbs often. The commands below are how you can invoke these actions.
+```sh
+cargo run --bin sui-indexer -- --database-url "<DATABASE_URL>" reset-database --force
+```
+
+## Steps to run locally (TiDB)
+
+### Prerequisites
+
+1. Install TiDB
+
+``` sh
+curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+```
+
+2. Install a compatible version of MySQL (At the time of writing, this is MySQL 8.0 -- note that 8.3 is incompatible).
+
+``` sh
+brew install [email protected]
+```
+
+3. Install a version of `diesel_cli` that supports MySQL (and probably also Postgres). This version of the CLI needs to be built against the version of MySQL that was installed in the previous step (compatible with the local installation of TiDB, 8.0.37 at time of writing).
+
+``` sh
+MYSQLCLIENT_LIB_DIR=/opt/homebrew/Cellar/[email protected]/8.0.37/lib/ cargo install diesel_cli --no-default-features --features postgres --features mysql --force
+```
+
+### Run the indexer
+
+1.Run TiDB
+
+```sh
+tiup playground
+```
+
+2.Verify tidb is running by connecting to it using the mysql client, create database `test`
+
+```sh
+mysql --comments --host 127.0.0.1 --port 4000 -u root
+create database test;
+```
+
+3.DB setup, under `sui/crates/sui-indexer` run:
+
+```sh
+# an example DATABASE_URL is "mysql://root:[email protected]:4000/test"
+diesel setup --database-url="<DATABASE_URL>" --migration-dir='migrations/mysql'
+diesel database reset --database-url="<DATABASE_URL>" --migration-dir='migrations/mysql'
+```
+
+Note that you need an existing database for this to work. Using the DATABASE_URL example in the comment of the previous code, replace `test` with the name of your database.
+4. Run indexer as a writer, which pulls data from fullnode and writes data to DB
+
+```sh
+# Change the RPC_CLIENT_URL to http://0.0.0.0:9000 to run indexer against local validator & fullnode
+cargo run --bin sui-indexer --features mysql-feature --no-default-features -- --db-url "<DATABASE_URL>" --rpc-client-url "https://fullnode.devnet.sui.io:443" --fullnode-sync-worker --reset-db
+```
+
+### Extending the indexer
+
+To add a new table, run `diesel migration generate your_table_name`, and modify the newly created `up.sql` and `down.sql` files.
+
+You would apply the migration with `diesel migration run`, and run the script in `./scripts/generate_indexer_schema.sh` to update the `schema.rs` file.

crates/mvr-indexer/diesel.toml

@@ -0,0 +1,8 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema/pg.rs"
+
+[migrations_directory]
+dir = "migrations/pg"

crates/mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

crates/mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

crates/mvr-indexer/migrations/pg/2023-08-19-044020_events/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE IF EXISTS events;

crates/mvr-indexer/migrations/pg/2023-08-19-044020_events/up.sql

@@ -0,0 +1,26 @@
+-- TODO: modify queries in indexer reader to take advantage of the new indices
+CREATE TABLE events
+(
+    tx_sequence_number          BIGINT       NOT NULL,
+    event_sequence_number       BIGINT       NOT NULL,
+    transaction_digest          bytea        NOT NULL,
+    -- array of SuiAddress in bytes. All signers of the transaction.
+    senders                     bytea[]      NOT NULL,
+    -- bytes of the entry package ID. Notice that the package and module here
+    -- are the package and module of the function that emitted the event, diffrent
+    -- from the package and module of the event type.
+    package                     bytea        NOT NULL,
+    -- entry module name
+    module                      text         NOT NULL,
+    -- StructTag in Display format, fully qualified including type parameters
+    event_type                  text         NOT NULL,
+    -- timestamp of the checkpoint when the event was emitted
+    timestamp_ms                BIGINT       NOT NULL,
+    -- bcs of the Event contents (Event.contents)
+    bcs                         BYTEA        NOT NULL,
+    PRIMARY KEY(tx_sequence_number, event_sequence_number)
+) PARTITION BY RANGE (tx_sequence_number);
+CREATE TABLE events_partition_0 PARTITION OF events FOR VALUES FROM (0) TO (MAXVALUE);
+CREATE INDEX events_package ON events (package, tx_sequence_number, event_sequence_number);
+CREATE INDEX events_package_module ON events (package, module, tx_sequence_number, event_sequence_number);
+CREATE INDEX events_event_type ON events (event_type text_pattern_ops, tx_sequence_number, event_sequence_number);

crates/mvr-indexer/migrations/pg/2023-08-19-044023_objects/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS objects;
+DROP TABLE IF EXISTS objects_history;
+DROP TABLE IF EXISTS objects_snapshot;