Add dev resources

deps.edn

@@ -16,6 +16,9 @@
         com.xtdb/xtdb-jdbc {:mvn/version "1.21.0"}
         com.xtdb/xtdb-s3 {:mvn/version "1.21.0"}
 
+        mysql/mysql-connector-java {:mvn/version "8.0.29"}
+        diehard/diehard {:mvn/version "0.11.3"}
+
         ;; Jetty
         ring/ring-jetty-adapter {:mvn/version "1.9.5"}
 
@@ -88,7 +91,7 @@
  :aliases
  {:dev
   {:extra-paths ["dev" "test"]
-   :extra-deps { ;; Convenience libraries made available during development
+   :extra-deps {;; Convenience libraries made available during development
                 org.clojure/test.check {:mvn/version "1.1.1"}
                 nrepl/nrepl {:mvn/version "0.9.0"}
                 org.clojure/alpha.spec {:git/url "https://github.com/clojure/spec-alpha2.git"

dev/config.edn

@@ -0,0 +1,35 @@
+{
+ ;; Used by bin/site to know where to send HTTP API requests.
+ :juxt.site.alpha/base-uri "http://localhost:5509"
+
+ :ig/system
+ {:juxt.site.alpha.db/xt-node
+  {
+   :xtdb.http-server/server {:port 5511}
+   :xtdb.rocksdb/block-cache {:xtdb/module xtdb.rocksdb/->lru-block-cache
+                              :cache-size 1600000000}
+   :xtdb/tx-log
+   {:kv-store {:xtdb/module xtdb.rocksdb/->kv-store
+               :db-dir "db/txes"}}
+
+   :xtdb/document-store
+   {:kv-store {:xtdb/module xtdb.rocksdb/->kv-store
+               :db-dir "db/docs"}}
+
+   :xtdb/index-store
+   {:kv-store {:xtdb/module xtdb.rocksdb/->kv-store
+               :db-dir "db/idxs"}}}
+
+  :juxt.site.alpha.server/server
+  {:juxt.site.alpha/xt-node #ig/ref :juxt.site.alpha.db/xt-node
+   :juxt.site.alpha/port 5509
+
+   ;; Really, this is the canoncial-uri prefix where /_site exists.
+   :juxt.site.alpha/base-uri #ref [:juxt.site.alpha/base-uri]
+
+   :juxt.site.alpha/dynamic? #profile {:dev true :prod false}}
+
+  :juxt.site.alpha.nrepl/server
+  {:juxt.site.alpha/port 5510}
+}
+}

src/juxt/site/alpha/db.clj

@@ -4,11 +4,56 @@
   (:require
    [xtdb.api :as xt]
    [integrant.core :as ig]
-   [clojure.tools.logging :as log]))
+   [clojure.tools.logging :as log]
+   [diehard.core :as dh])
+  (:import
+   java.time.Duration
+   software.amazon.awssdk.services.s3.S3AsyncClient
+   software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient))
+
+
+
+(def s3-configurator
+  (reify xtdb.s3.S3Configurator
+    (makeClient [_this]
+      (.. (S3AsyncClient/builder)
+          (httpClientBuilder
+            (.. (NettyNioAsyncHttpClient/builder)
+                (connectionAcquisitionTimeout (Duration/ofSeconds 600))
+                (maxConcurrency (Integer. 100))
+                (maxPendingConnectionAcquires (Integer. 10000))))
+          (build)))))
+
+(defn- start-node
+  [config]
+  (dh/with-retry
+    {:retry-if
+     (fn [_ ex]
+       (= "incomplete checkpoint restore"
+          (ex-message ex)))
+     :max-retries 3
+     :on-failed-attempt
+     (fn [_ ex]
+       (log/warn ex "Couldn't complete checkpoint restore"))
+     :on-failure
+     (fn [_ ex]
+       (log/error ex "Checkpoint restore failed"))}
+    (xt/start-node config)))
 
 (defmethod ig/init-key ::xt-node [_ xtdb-opts]
-  (log/info "Starting XT node")
-  (xt/start-node xtdb-opts))
+  (log/info "Starting XT node ...")
+
+  (let [config (update-in xtdb-opts
+                          [:xtdb/index-store :kv-store :checkpointer :store]
+                          assoc :configurator (constantly s3-configurator))
+        node (start-node config)]
+    ;; we need to make sure the tx-ingester has caught up before
+    ;; declaring the node up
+    (->>
+     (xt/submit-tx node [[::xt/put {:xt/id :tx-ingester-synced!}]])
+     (xt/await-tx node))
+    (log/info "... XT node started!")
+    node))
 
 (defmethod ig/halt-key! ::xt-node [_ node]
   (.close node)