From 4631ef8168ab70dae918ea5ba7c45b5665f96525 Mon Sep 17 00:00:00 2001
From: James Kerr <kerr@hey.com>
Date: Wed, 24 Jan 2024 15:05:32 -0800
Subject: [PATCH] Export to Pool Looking Good

---
 .../sidebar/queries-section/index.tsx         |  1 +
 apps/zui/src/core/loader/types.ts             | 19 -----------
 apps/zui/src/domain/legacy-ops/messages.ts    |  2 +-
 apps/zui/src/domain/loads/default-loaders.ts  | 15 ++++++++
 apps/zui/src/domain/loads/file-loader.ts      |  4 +--
 apps/zui/src/domain/loads/load-context.ts     | 15 ++++++--
 apps/zui/src/domain/loads/load-model.ts       |  6 ++++
 apps/zui/src/domain/loads/load-ref.ts         |  4 ++-
 .../zui/src/domain/loads/operations/cancel.ts |  4 +--
 apps/zui/src/domain/loads/plugin-api.ts       | 27 ++++++---------
 apps/zui/src/domain/loads/query-loader.ts     | 34 +++++++++++++++++++
 apps/zui/src/domain/loads/types.ts            | 23 ++++++++++---
 apps/zui/src/domain/pools/plugin-api.ts       |  6 ++--
 .../zui/src/domain/results/handlers/export.ts | 16 ++++++---
 apps/zui/src/domain/results/operations.ts     | 21 +++++++-----
 apps/zui/src/domain/results/utils.ts          |  4 ---
 apps/zui/src/js/api/pools/pools-api.ts        |  2 +-
 apps/zui/src/js/components/Toaster.tsx        |  3 ++
 apps/zui/src/js/state/Loads/types.ts          |  1 +
 apps/zui/src/plugins/brimcap/loader.ts        |  2 +-
 apps/zui/src/views/load-pane/form.tsx         |  2 +-
 apps/zui/src/views/load-pane/index.tsx        |  2 +-
 apps/zui/src/views/load-pane/results.tsx      |  1 +
 apps/zui/src/views/pool-page/recent-loads.tsx |  2 +-
 apps/zui/src/views/session-page/footer.tsx    |  1 +
 25 files changed, 145 insertions(+), 72 deletions(-)
 delete mode 100644 apps/zui/src/core/loader/types.ts
 create mode 100644 apps/zui/src/domain/loads/default-loaders.ts

diff --git a/apps/zui/src/app/features/sidebar/queries-section/index.tsx b/apps/zui/src/app/features/sidebar/queries-section/index.tsx
index f8fc789b34..aab06dfca0 100644
--- a/apps/zui/src/app/features/sidebar/queries-section/index.tsx
+++ b/apps/zui/src/app/features/sidebar/queries-section/index.tsx
@@ -19,6 +19,7 @@ export function QueriesSection() {
       </Content>
       <Toolbar>
         <ToolbarTabs
+          name="querySource"
           options={[
             {
               label: "Local",
diff --git a/apps/zui/src/core/loader/types.ts b/apps/zui/src/core/loader/types.ts
deleted file mode 100644
index 797f8d7a33..0000000000
--- a/apps/zui/src/core/loader/types.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import {LoadFormat} from "@brimdata/zed-js"
-
-export type LoadOptions = {
-  windowId: string
-  lakeId: string
-  poolId: string
-  branch: string
-  files: string[]
-  shaper: string
-  format?: LoadFormat
-  author: string
-  body: string
-}
-
-export interface Loader {
-  when(): PromiseLike<boolean> | boolean
-  run(): PromiseLike<void> | void
-  rollback?(): PromiseLike<void> | void
-}
diff --git a/apps/zui/src/domain/legacy-ops/messages.ts b/apps/zui/src/domain/legacy-ops/messages.ts
index 53abf8637f..ced84c05af 100644
--- a/apps/zui/src/domain/legacy-ops/messages.ts
+++ b/apps/zui/src/domain/legacy-ops/messages.ts
@@ -6,7 +6,6 @@ import {Config} from "../configurations/plugin-api"
 import {CompiledCorrelation} from "../correlations/plugin-api"
 import {MenuItem} from "src/core/menu"
 import {AnyAction} from "@reduxjs/toolkit"
-import {LoadOptions} from "src/core/loader/types"
 import {MainArgs} from "src/electron/run-main/args"
 import {
   MenuItemConstructorOptions,
@@ -20,6 +19,7 @@ import {
 import {SearchAppMenuState} from "src/electron/windows/search/app-menu"
 import {Pool} from "src/app/core/pools/pool"
 import {Command} from "src/app/commands/command"
+import {LoadOptions} from "../loads/types"
 
 export type LegacyOperations = {
   autosaveOp: (windowId: string, windowState: State) => void
diff --git a/apps/zui/src/domain/loads/default-loaders.ts b/apps/zui/src/domain/loads/default-loaders.ts
new file mode 100644
index 0000000000..1298b8bd0f
--- /dev/null
+++ b/apps/zui/src/domain/loads/default-loaders.ts
@@ -0,0 +1,15 @@
+import {FileLoader} from "./file-loader"
+import {LoadContext} from "./load-context"
+import {QueryLoader} from "./query-loader"
+import {LoaderRef} from "./types"
+
+export const DEFAULT_LOADERS = [
+  {
+    name: "fileLoader",
+    initialize: (ctx: LoadContext) => new FileLoader(ctx),
+  },
+  {
+    name: "queryLoader",
+    initialize: (ctx: LoadContext) => new QueryLoader(ctx),
+  },
+] as LoaderRef[]
diff --git a/apps/zui/src/domain/loads/file-loader.ts b/apps/zui/src/domain/loads/file-loader.ts
index 89c20ce1d1..220eb96de9 100644
--- a/apps/zui/src/domain/loads/file-loader.ts
+++ b/apps/zui/src/domain/loads/file-loader.ts
@@ -6,11 +6,11 @@ import {createReadableStream} from "src/core/zq"
 import {throttle} from "lodash"
 import {errorToString} from "src/util/error-to-string"
 
-export class DefaultLoader implements Loader {
+export class FileLoader implements Loader {
   constructor(private ctx: LoadContext) {}
 
   when() {
-    return true
+    return this.ctx.files.length > 0
   }
 
   async run() {
diff --git a/apps/zui/src/domain/loads/load-context.ts b/apps/zui/src/domain/loads/load-context.ts
index a764262dfd..1fd5fdc3dc 100644
--- a/apps/zui/src/domain/loads/load-context.ts
+++ b/apps/zui/src/domain/loads/load-context.ts
@@ -26,7 +26,14 @@ export class LoadContext {
     this.window.loadsInProgress++
     this.main.abortables.add({id: this.id, abort: () => this.ctl.abort()})
     this.main.dispatch(
-      Loads.create(createLoadRef(this.id, this.opts.poolId, this.opts.files))
+      Loads.create(
+        createLoadRef(
+          this.id,
+          this.opts.poolId,
+          this.opts.files,
+          this.opts.query
+        )
+      )
     )
   }
 
@@ -74,7 +81,11 @@ export class LoadContext {
   }
 
   get files() {
-    return this.opts.files
+    return this.opts.files || []
+  }
+
+  get query() {
+    return this.opts.query
   }
 
   get lakeId() {
diff --git a/apps/zui/src/domain/loads/load-model.ts b/apps/zui/src/domain/loads/load-model.ts
index 5e5c67f372..cd72b0b7a6 100644
--- a/apps/zui/src/domain/loads/load-model.ts
+++ b/apps/zui/src/domain/loads/load-model.ts
@@ -8,7 +8,13 @@ export class LoadModel {
     return this.ref.id
   }
 
+  get title() {
+    if (this.ref.files.length) return this.humanizeFiles
+    else return this.ref.query
+  }
+
   get humanizeFiles() {
+    if (!this.ref.files) return "No files"
     return this.ref.files.map(basename).join(", ")
   }
 
diff --git a/apps/zui/src/domain/loads/load-ref.ts b/apps/zui/src/domain/loads/load-ref.ts
index d0d27796c8..5ddc976fb7 100644
--- a/apps/zui/src/domain/loads/load-ref.ts
+++ b/apps/zui/src/domain/loads/load-ref.ts
@@ -3,12 +3,14 @@ import {LoadReference} from "src/js/state/Loads/types"
 export function createLoadRef(
   id: string,
   poolId: string,
-  files: string[]
+  files: string[],
+  query: string
 ): LoadReference {
   return {
     id,
     poolId,
     progress: 0,
+    query,
     files,
     startedAt: new Date().toISOString(),
     finishedAt: null,
diff --git a/apps/zui/src/domain/loads/operations/cancel.ts b/apps/zui/src/domain/loads/operations/cancel.ts
index a010e46922..7220ad71b7 100644
--- a/apps/zui/src/domain/loads/operations/cancel.ts
+++ b/apps/zui/src/domain/loads/operations/cancel.ts
@@ -4,7 +4,7 @@ import {createOperation} from "src/core/operations"
 
 export const cancel = createOperation(
   "loads.cancel",
-  (ctx, poolId: string, files: string[]) => {
-    loads.emit("abort", createLoadRef("new", poolId, files))
+  (ctx, poolId: string, files: string[], query: string) => {
+    loads.emit("abort", createLoadRef("new", poolId, files, query))
   }
 )
diff --git a/apps/zui/src/domain/loads/plugin-api.ts b/apps/zui/src/domain/loads/plugin-api.ts
index 87d8b0290d..7d39bc4d7a 100644
--- a/apps/zui/src/domain/loads/plugin-api.ts
+++ b/apps/zui/src/domain/loads/plugin-api.ts
@@ -1,20 +1,11 @@
-import {DefaultLoader} from "./file-loader"
 import {LoadContext} from "./load-context"
-import {Loader} from "src/core/loader/types"
 import Loads from "src/js/state/Loads"
 import {select} from "src/core/main/select"
 import {TypedEmitter} from "src/util/typed-emitter"
-import {LoadReference} from "src/js/state/Loads/types"
+import {DEFAULT_LOADERS} from "./default-loaders"
+import {LoadEvents, Loader, LoaderRef} from "./types"
 
-type Events = {
-  success: (load: LoadReference) => void
-  abort: (load: LoadReference) => void
-  error: (load: LoadReference) => void
-}
-
-type LoaderRef = {name: string; initialize: (ctx: LoadContext) => Loader}
-
-export class LoadsApi extends TypedEmitter<Events> {
+export class LoadsApi extends TypedEmitter<LoadEvents> {
   private list: LoaderRef[] = []
 
   addLoader(name: string, initialize: (ctx: LoadContext) => Loader) {
@@ -22,11 +13,15 @@ export class LoadsApi extends TypedEmitter<Events> {
   }
 
   async initialize(context: LoadContext) {
-    for (const ref of this.list) {
-      const customLoader = ref.initialize(context)
-      if (await customLoader.when()) return customLoader
+    for (const {initialize} of this.loaders) {
+      const loader = initialize(context)
+      if (await loader.when()) return loader
     }
-    return new DefaultLoader(context)
+    throw new Error("Loader not found")
+  }
+
+  private get loaders() {
+    return [...this.list, ...DEFAULT_LOADERS]
   }
 
   get all() {
diff --git a/apps/zui/src/domain/loads/query-loader.ts b/apps/zui/src/domain/loads/query-loader.ts
index e69de29bb2..3fd47fdf40 100644
--- a/apps/zui/src/domain/loads/query-loader.ts
+++ b/apps/zui/src/domain/loads/query-loader.ts
@@ -0,0 +1,34 @@
+import {LoadContext} from "./load-context"
+import {Loader} from "./types"
+
+export class QueryLoader implements Loader {
+  constructor(private ctx: LoadContext) {}
+
+  when() {
+    return this.ctx.files.length === 0 && !!this.ctx.query
+  }
+
+  async run() {
+    this.ctx.setProgress(0)
+    const client = await this.ctx.createClient()
+    const res = await client.query(this.loadQuery).then((r) => r.js())
+    console.log(res)
+    this.ctx.setProgress(1)
+  }
+
+  private get loadQuery() {
+    // This is the load op syntax
+    // load <pool>[@<branch>] [author <author>] [message <message>] [meta <meta>]
+    return [
+      this.ctx.query,
+      "| load",
+      this.ctx.poolId + "@" + this.ctx.branch,
+      "author " + JSON.stringify(this.ctx.author),
+      "message " + JSON.stringify(this.ctx.body),
+    ].join(" ")
+  }
+}
+
+export function addLoad(query: string, poolId) {
+  return query + " | load " + poolId
+}
diff --git a/apps/zui/src/domain/loads/types.ts b/apps/zui/src/domain/loads/types.ts
index bc66980f9e..483445f999 100644
--- a/apps/zui/src/domain/loads/types.ts
+++ b/apps/zui/src/domain/loads/types.ts
@@ -1,20 +1,33 @@
 import {LoadFormat} from "@brimdata/zed-js"
 import {LoadContext} from "./load-context"
+import {LoadReference} from "src/js/state/Loads/types"
 
 export type LoadOptions = {
   windowId: string
   lakeId: string
   poolId: string
-  branch: string
+  branch?: string
   files: string[]
-  shaper: string
+  query?: string
+  shaper?: string
   format?: LoadFormat
   author: string
   body: string
 }
 
 export interface Loader {
-  when(context: LoadContext): PromiseLike<boolean> | boolean
-  run(context: LoadContext): PromiseLike<void> | void
-  rollback(context: LoadContext): PromiseLike<void> | void
+  when(): PromiseLike<boolean> | boolean
+  run(): PromiseLike<void> | void
+  rollback?(): PromiseLike<void> | void
+}
+
+export type LoadEvents = {
+  success: (load: LoadReference) => void
+  abort: (load: LoadReference) => void
+  error: (load: LoadReference) => void
+}
+
+export type LoaderRef = {
+  name: string
+  initialize: (ctx: LoadContext) => Loader
 }
diff --git a/apps/zui/src/domain/pools/plugin-api.ts b/apps/zui/src/domain/pools/plugin-api.ts
index 96474f9c58..109becd642 100644
--- a/apps/zui/src/domain/pools/plugin-api.ts
+++ b/apps/zui/src/domain/pools/plugin-api.ts
@@ -6,10 +6,11 @@ import {loads, window} from "src/zui"
 import * as ops from "./operations"
 import {LoadContext} from "src/domain/loads/load-context"
 import {syncPoolOp} from "src/electron/ops/sync-pool-op"
-import {LoadOptions} from "src/core/loader/types"
 import {getMainObject} from "src/core/main"
 import {TypedEmitter} from "src/util/typed-emitter"
 import {call} from "src/util/call"
+import {LoadOptions} from "../loads/types"
+import {debug} from "src/core/log"
 
 type Events = {
   create: (event: {pool: Pool}) => void
@@ -29,7 +30,7 @@ export class PoolsApi extends TypedEmitter<Events> {
   }
 
   async create(name: string, opts: Partial<CreatePoolOpts> = {}) {
-    const id = await ops.create(window.lakeId, name, opts)
+    const id = await ops.create(name, opts)
     return this.get(id)
   }
 
@@ -41,6 +42,7 @@ export class PoolsApi extends TypedEmitter<Events> {
     const main = getMainObject()
     const context = new LoadContext(main, opts)
     const loader = await loads.initialize(context)
+    debug("Using Loader", loader)
     try {
       await context.setup()
       await loader.run()
diff --git a/apps/zui/src/domain/results/handlers/export.ts b/apps/zui/src/domain/results/handlers/export.ts
index 3fc6c4cf2d..684a5ec527 100644
--- a/apps/zui/src/domain/results/handlers/export.ts
+++ b/apps/zui/src/domain/results/handlers/export.ts
@@ -15,7 +15,12 @@ export const exportToPool = createHandler(
     const query = getExportQuery(null)
     try {
       const poolId = await getOrCreatePool(data)
-      const promise = invoke("results.exportToPool", query, poolId)
+      const promise = invoke(
+        "results.exportToPool",
+        query,
+        poolId,
+        globalThis.windowId
+      )
       toast.promise(promise, {
         loading: "Exporting to pool...",
         success: "Export to pool complete",
@@ -58,9 +63,12 @@ export const exportToFile = createHandler(
 export const exportToClipboard = createHandler(
   async (ctx, format: ResponseFormat) => {
     const query = getExportQuery(format)
-    await ctx.invoke("results.copyToClipboard", query, format)
-
-    ctx.toast.success(`Copied ${format} data to clipboard.`)
+    const promise = ctx.invoke("results.copyToClipboard", query, format)
+    ctx.toast.promise(promise, {
+      loading: "Copying...",
+      error: (e) => errorToString(e),
+      success: `Copied ${format} data to clipboard.`,
+    })
   }
 )
 
diff --git a/apps/zui/src/domain/results/operations.ts b/apps/zui/src/domain/results/operations.ts
index d45b2475d8..dd057a7a29 100644
--- a/apps/zui/src/domain/results/operations.ts
+++ b/apps/zui/src/domain/results/operations.ts
@@ -3,10 +3,8 @@ import {ResponseFormat} from "@brimdata/zed-js"
 import fs from "fs"
 import {pipeline} from "stream"
 import util from "util"
-import {lake} from "src/zui"
+import {lake, pools} from "src/zui"
 import {clipboard} from "electron"
-import {addLoad} from "./utils"
-import { debug } from "src/core/log"
 
 const pipe = util.promisify(pipeline)
 
@@ -46,13 +44,18 @@ export const copyToClipboard = createOperation(
 
 export const exportToPool = createOperation(
   "results.exportToPool",
-  async (ctx, query: string, poolId) => {
+  async (ctx, query: string, poolId: string, windowId: string) => {
     if (!poolId) throw new Error("Argument missing: poolId")
     if (!query) throw new Error("Argument missing: query")
-    const loadQuery = addLoad(query, poolId)
-    debug(loadQuery)
-    const res = await lake.query(loadQuery)
-    const result = await res.js()
-    console.log(result)
+    return pools.load({
+      windowId,
+      lakeId: lake.id,
+      poolId,
+      branch: "main",
+      query,
+      files: [],
+      author: "Zui",
+      body: "Export to pool",
+    })
   }
 )
diff --git a/apps/zui/src/domain/results/utils.ts b/apps/zui/src/domain/results/utils.ts
index af787212e5..1b358a52be 100644
--- a/apps/zui/src/domain/results/utils.ts
+++ b/apps/zui/src/domain/results/utils.ts
@@ -9,7 +9,3 @@ export function cutColumns(query: string, names: string[]) {
 export function addFuse(query: string) {
   return query + " | fuse"
 }
-
-export function addLoad(query: string, poolId) {
-  return query + " | load " + poolId
-}
diff --git a/apps/zui/src/js/api/pools/pools-api.ts b/apps/zui/src/js/api/pools/pools-api.ts
index e3b3473678..93e0103578 100644
--- a/apps/zui/src/js/api/pools/pools-api.ts
+++ b/apps/zui/src/js/api/pools/pools-api.ts
@@ -48,7 +48,7 @@ export class PoolsApi extends ApiDomain {
   }
 
   async create(name: string, opts: Partial<CreatePoolOpts> = {}) {
-    const id = await invoke("pools.create", this.lakeId, name, opts)
+    const id = await invoke("pools.create", name, opts)
     await this.sync(id)
     return id
   }
diff --git a/apps/zui/src/js/components/Toaster.tsx b/apps/zui/src/js/components/Toaster.tsx
index 5544db32c6..6b187f6e37 100644
--- a/apps/zui/src/js/components/Toaster.tsx
+++ b/apps/zui/src/js/components/Toaster.tsx
@@ -15,6 +15,9 @@ const Toaster = () => {
         success: {
           duration: 6000,
         },
+        error: {
+          duration: 15_000,
+        },
       }}
     />
   )
diff --git a/apps/zui/src/js/state/Loads/types.ts b/apps/zui/src/js/state/Loads/types.ts
index 15739adcfb..5d6f18277c 100644
--- a/apps/zui/src/js/state/Loads/types.ts
+++ b/apps/zui/src/js/state/Loads/types.ts
@@ -5,6 +5,7 @@ export type LoadReference = {
   poolId: string
   progress: number
   files: string[]
+  query: string
   startedAt: string
   finishedAt: string | null
   abortedAt: string | null
diff --git a/apps/zui/src/plugins/brimcap/loader.ts b/apps/zui/src/plugins/brimcap/loader.ts
index 1b0f07832d..31061223a9 100644
--- a/apps/zui/src/plugins/brimcap/loader.ts
+++ b/apps/zui/src/plugins/brimcap/loader.ts
@@ -1,5 +1,4 @@
 import fs from "fs"
-import {Loader} from "src/core/loader/types"
 import {LoadContext} from "src/domain/loads/load-context"
 import {isPcap} from "./packets/is-pcap"
 import {loads} from "src/zui"
@@ -10,6 +9,7 @@ import {createAnalyzeProcess, monitorAnalyzeProgress} from "./analyze"
 import {createCli} from "./cli"
 import errors from "src/js/errors"
 import {errorToString} from "src/util/error-to-string"
+import {Loader} from "src/domain/loads/types"
 
 class BrimcapLoader implements Loader {
   constructor(private ctx: LoadContext, private root: string) {}
diff --git a/apps/zui/src/views/load-pane/form.tsx b/apps/zui/src/views/load-pane/form.tsx
index 71a2087379..44c41da8b0 100644
--- a/apps/zui/src/views/load-pane/form.tsx
+++ b/apps/zui/src/views/load-pane/form.tsx
@@ -145,7 +145,7 @@ export function Form(props: {
           </div>
 
           <Show when={watch("poolId") == "new"}>
-            <PoolForm register={register} />
+            <PoolForm />
           </Show>
           <div className="field">
             <label htmlFor="author">Author</label>
diff --git a/apps/zui/src/views/load-pane/index.tsx b/apps/zui/src/views/load-pane/index.tsx
index 311bba4915..6d651aba9e 100644
--- a/apps/zui/src/views/load-pane/index.tsx
+++ b/apps/zui/src/views/load-pane/index.tsx
@@ -83,7 +83,7 @@ function Pane(props: {onClose: any}) {
   const onCancel = () => {
     const files = select(LoadDataForm.getFiles)
     const poolId = select(LoadDataForm.getPoolId)
-    invoke("loads.cancel", poolId, files)
+    invoke("loads.cancel", poolId, files, "")
     debut.exit()
   }
 
diff --git a/apps/zui/src/views/load-pane/results.tsx b/apps/zui/src/views/load-pane/results.tsx
index a61c0ac7f3..a75c916943 100644
--- a/apps/zui/src/views/load-pane/results.tsx
+++ b/apps/zui/src/views/load-pane/results.tsx
@@ -83,6 +83,7 @@ function Toolbar(props: {
       </div>
       <div className={styles.middle}>
         <ToolbarTabs
+          name="resultsView"
           onlyIcon={props.smallWidth}
           options={[
             {
diff --git a/apps/zui/src/views/pool-page/recent-loads.tsx b/apps/zui/src/views/pool-page/recent-loads.tsx
index 7337eef1da..ae577d5358 100644
--- a/apps/zui/src/views/pool-page/recent-loads.tsx
+++ b/apps/zui/src/views/pool-page/recent-loads.tsx
@@ -36,7 +36,7 @@ export function RecentLoads(props: {id: string}) {
           .map((load) => (
             <Job
               key={load.id}
-              name={load.humanizeFiles}
+              name={load.title}
               message={statusMessage(load)}
               status={load.status}
               progress={load.progress}
diff --git a/apps/zui/src/views/session-page/footer.tsx b/apps/zui/src/views/session-page/footer.tsx
index 996679fda1..f954250beb 100644
--- a/apps/zui/src/views/session-page/footer.tsx
+++ b/apps/zui/src/views/session-page/footer.tsx
@@ -44,6 +44,7 @@ export function Footer() {
   return (
     <footer className={styles.footer}>
       <ToolbarTabs
+        name="resultsView"
         onlyIcon={true}
         options={[
           {