Redis and logging improvements (#30)

* Improved redis interface

* Distributed lock implementation for redis

* GlobalLog wait for collector for 10 seconds, don't just error when not available straight away.

* Otlp metric network/client impact reduced in js

js/bitbazaar/log/log.ts

@@ -252,7 +252,7 @@ class GlobalLog {
                         ...baseExporterConfig,
                         url: `${otlp.endpoint}/v1/metrics`,
                     }),
-                    exportIntervalMillis: 1000,
+                    exportIntervalMillis: 60000, // Haven't found a way for it to not send when no metrics yet, so changing from 1s to 60s to not bloat the network logs of a client.
                 }),
             ],
         });

js/package.json

@@ -40,16 +40,16 @@
     "vite-plugin-checker": "0.6.2",
     "vite-plugin-inspect": "0.8.1",
     "vite-plugin-pwa": "0.17.4",
-    "@opentelemetry/api": "^1.7.0",
-    "@opentelemetry/context-zone": "^1.21.0",
-    "@opentelemetry/exporter-logs-otlp-http": "^0.48.0",
-    "@opentelemetry/exporter-metrics-otlp-http": "^0.48.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.48.0",
-    "@opentelemetry/instrumentation-document-load": "^0.35.0",
-    "@opentelemetry/instrumentation-fetch": "^0.48.0",
-    "@opentelemetry/instrumentation-user-interaction": "^0.35.0",
-    "@opentelemetry/sdk-logs": "^0.48.0",
-    "@opentelemetry/sdk-metrics": "^1.21.0",
-    "@opentelemetry/sdk-trace-web": "^1.21.0"
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/context-zone": "^1.23.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.50.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.50.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
+    "@opentelemetry/instrumentation-document-load": "^0.37.0",
+    "@opentelemetry/instrumentation-fetch": "^0.50.0",
+    "@opentelemetry/instrumentation-user-interaction": "^0.37.0",
+    "@opentelemetry/sdk-logs": "^0.50.0",
+    "@opentelemetry/sdk-metrics": "^1.23.0",
+    "@opentelemetry/sdk-trace-web": "^1.23.0"
   }
 }

opencollector.yaml

@@ -32,7 +32,7 @@ exporters:
       stream-name: default
   # Writes all opentelemetry logs, traces, metrics to a file, useful for testing:
   file/debug_file_writing:
-    path: /home/runner/work/bitbazaar/bitbazaar/logs/otlp_telemetry_out.log
+    path: /Users/zak/z/code/bitbazaar/logs/otlp_telemetry_out.log
     rotation:
       max_megabytes: 10
       max_days: 3

rust/Cargo.toml

@@ -23,7 +23,15 @@ all-features = true
 log-filter = ["dep:regex"]
 timing = ['dep:comfy-table', 'dep:chrono']
 cli = ['dep:normpath', 'dep:conch-parser', 'dep:homedir', 'dep:chrono', 'dep:strum']
-redis = ['dep:deadpool-redis', 'dep:redis', 'dep:sha1_smol', 'dep:serde_json']
+redis = [
+  'dep:deadpool-redis',
+  'dep:redis',
+  'dep:sha1_smol',
+  'dep:serde_json',
+  'dep:rand',
+  'dep:futures',
+  'dep:futures-timer',
+]
 opentelemetry-grpc = [
   'dep:tracing-log',
   'dep:opentelemetry-appender-tracing',
@@ -65,6 +73,9 @@ colored = '2'
 chrono = { version = '0.4', optional = true }
 strum = { version = "0.25", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
+rand = { version = "0.8", optional = true }
+futures = { version = "0.3", optional = true }
+futures-timer = { version = "3", optional = true }
 
 # FEAT: log-filter:
 regex = { version = '1', optional = true }
@@ -78,8 +89,8 @@ conch-parser = { version = "0.1.1", optional = true }
 homedir = { version = "0.2", optional = true }
 
 # FEAT: redis:
-deadpool-redis = { version = "0.14", features = ["rt_tokio_1"], optional = true }
-redis = { version = "0.24", default-features = false, features = ["aio", "json"], optional = true }
+deadpool-redis = { version = "0.15", features = ["rt_tokio_1"], optional = true }
+redis = { version = "0.25", default-features = false, features = ["aio", "json"], optional = true }
 sha1_smol = { version = "1.0", optional = true }
 
 # FEAT: opentelemetry-(grpc|http):

rust/bitbazaar/log/global_log/setup.rs

@@ -175,9 +175,21 @@ pub fn builder_into_global_log(builder: GlobalLogBuilder) -> Result<GlobalLog, A
                 if let Some(port) = otlp.grpc_port {
                     use opentelemetry_otlp::{new_exporter, WithExportConfig};
 
-                    if !crate::misc::is_tcp_port_listening("localhost", port)? {
-                        return Err(anyerr!("Can't connect to open telemetry collector on local port {}. Are you sure it's running?", port));
+                    // Spinlock up to 10 seconds until the collector is listening, important not to lose startup logs, don't want to continue until we know the collector can receive logs.
+                    let wait_start = std::time::Instant::now();
+                    let mut found_collector = false;
+                    while wait_start.elapsed() < std::time::Duration::from_secs(10) {
+                        if crate::misc::is_tcp_port_listening("localhost", port)? {
+                            found_collector = true;
+                            break;
+                        }
+                        // Don't want this to delay startup otherwise, so very short waits:
+                        std::thread::sleep(std::time::Duration::from_millis(5));
                     }
+                    if !found_collector {
+                        return Err(anyerr!("Can't connect to open telemetry collector on local port {} (waited for 10 seconds). Are you sure it's running?", port));
+                    }
+
                     let endpoint = format!("grpc://localhost:{}", port);
                     let get_exporter = || new_exporter().tonic().with_endpoint(&endpoint);
                     exporters.push((