Export to Pool Looking Good

apps/zui/src/app/features/sidebar/queries-section/index.tsx

@@ -19,6 +19,7 @@ export function QueriesSection() {
       </Content>
       <Toolbar>
         <ToolbarTabs
+          name="querySource"
           options={[
             {
               label: "Local",

apps/zui/src/domain/legacy-ops/messages.ts

@@ -6,7 +6,6 @@ import {Config} from "../configurations/plugin-api"
 import {CompiledCorrelation} from "../correlations/plugin-api"
 import {MenuItem} from "src/core/menu"
 import {AnyAction} from "@reduxjs/toolkit"
-import {LoadOptions} from "src/core/loader/types"
 import {MainArgs} from "src/electron/run-main/args"
 import {
   MenuItemConstructorOptions,
@@ -20,6 +19,7 @@ import {
 import {SearchAppMenuState} from "src/electron/windows/search/app-menu"
 import {Pool} from "src/app/core/pools/pool"
 import {Command} from "src/app/commands/command"
+import {LoadOptions} from "../loads/types"
 
 export type LegacyOperations = {
   autosaveOp: (windowId: string, windowState: State) => void

apps/zui/src/domain/loads/default-loaders.ts

@@ -0,0 +1,15 @@
+import {FileLoader} from "./file-loader"
+import {LoadContext} from "./load-context"
+import {QueryLoader} from "./query-loader"
+import {LoaderRef} from "./types"
+
+export const DEFAULT_LOADERS = [
+  {
+    name: "fileLoader",
+    initialize: (ctx: LoadContext) => new FileLoader(ctx),
+  },
+  {
+    name: "queryLoader",
+    initialize: (ctx: LoadContext) => new QueryLoader(ctx),
+  },
+] as LoaderRef[]

apps/zui/src/domain/loads/file-loader.ts

@@ -6,11 +6,11 @@ import {createReadableStream} from "src/core/zq"
 import {throttle} from "lodash"
 import {errorToString} from "src/util/error-to-string"
 
-export class DefaultLoader implements Loader {
+export class FileLoader implements Loader {
   constructor(private ctx: LoadContext) {}
 
   when() {
-    return true
+    return this.ctx.files.length > 0
   }
 
   async run() {

apps/zui/src/domain/loads/load-context.ts

@@ -26,7 +26,14 @@ export class LoadContext {
     this.window.loadsInProgress++
     this.main.abortables.add({id: this.id, abort: () => this.ctl.abort()})
     this.main.dispatch(
-      Loads.create(createLoadRef(this.id, this.opts.poolId, this.opts.files))
+      Loads.create(
+        createLoadRef(
+          this.id,
+          this.opts.poolId,
+          this.opts.files,
+          this.opts.query
+        )
+      )
     )
   }
 
@@ -74,7 +81,11 @@ export class LoadContext {
   }
 
   get files() {
-    return this.opts.files
+    return this.opts.files || []
+  }
+
+  get query() {
+    return this.opts.query
   }
 
   get lakeId() {

apps/zui/src/domain/loads/load-model.ts

@@ -8,7 +8,13 @@ export class LoadModel {
     return this.ref.id
   }
 
+  get title() {
+    if (this.ref.files.length) return this.humanizeFiles
+    else return this.ref.query
+  }
+
   get humanizeFiles() {
+    if (!this.ref.files) return "No files"
     return this.ref.files.map(basename).join(", ")
   }
 

apps/zui/src/domain/loads/load-ref.ts

@@ -3,12 +3,14 @@ import {LoadReference} from "src/js/state/Loads/types"
 export function createLoadRef(
   id: string,
   poolId: string,
-  files: string[]
+  files: string[],
+  query: string
 ): LoadReference {
   return {
     id,
     poolId,
     progress: 0,
+    query,
     files,
     startedAt: new Date().toISOString(),
     finishedAt: null,

apps/zui/src/domain/loads/operations/cancel.ts

@@ -4,7 +4,7 @@ import {createOperation} from "src/core/operations"
 
 export const cancel = createOperation(
   "loads.cancel",
-  (ctx, poolId: string, files: string[]) => {
-    loads.emit("abort", createLoadRef("new", poolId, files))
+  (ctx, poolId: string, files: string[], query: string) => {
+    loads.emit("abort", createLoadRef("new", poolId, files, query))
   }
 )

apps/zui/src/domain/loads/plugin-api.ts

@@ -1,32 +1,27 @@
-import {DefaultLoader} from "./file-loader"
 import {LoadContext} from "./load-context"
-import {Loader} from "src/core/loader/types"
 import Loads from "src/js/state/Loads"
 import {select} from "src/core/main/select"
 import {TypedEmitter} from "src/util/typed-emitter"
-import {LoadReference} from "src/js/state/Loads/types"
+import {DEFAULT_LOADERS} from "./default-loaders"
+import {LoadEvents, Loader, LoaderRef} from "./types"
 
-type Events = {
-  success: (load: LoadReference) => void
-  abort: (load: LoadReference) => void
-  error: (load: LoadReference) => void
-}
-
-type LoaderRef = {name: string; initialize: (ctx: LoadContext) => Loader}
-
-export class LoadsApi extends TypedEmitter<Events> {
+export class LoadsApi extends TypedEmitter<LoadEvents> {
   private list: LoaderRef[] = []
 
   addLoader(name: string, initialize: (ctx: LoadContext) => Loader) {
     this.list.push({name, initialize})
   }
 
   async initialize(context: LoadContext) {
-    for (const ref of this.list) {
-      const customLoader = ref.initialize(context)
-      if (await customLoader.when()) return customLoader
+    for (const {initialize} of this.loaders) {
+      const loader = initialize(context)
+      if (await loader.when()) return loader
     }
-    return new DefaultLoader(context)
+    throw new Error("Loader not found")
+  }
+
+  private get loaders() {
+    return [...this.list, ...DEFAULT_LOADERS]
   }
 
   get all() {

apps/zui/src/domain/loads/query-loader.ts

@@ -0,0 +1,34 @@
+import {LoadContext} from "./load-context"
+import {Loader} from "./types"
+
+export class QueryLoader implements Loader {
+  constructor(private ctx: LoadContext) {}
+
+  when() {
+    return this.ctx.files.length === 0 && !!this.ctx.query
+  }
+
+  async run() {
+    this.ctx.setProgress(0)
+    const client = await this.ctx.createClient()
+    const res = await client.query(this.loadQuery).then((r) => r.js())
+    console.log(res)
+    this.ctx.setProgress(1)
+  }
+
+  private get loadQuery() {
+    // This is the load op syntax
+    // load <pool>[@<branch>] [author <author>] [message <message>] [meta <meta>]
+    return [
+      this.ctx.query,
+      "| load",
+      this.ctx.poolId + "@" + this.ctx.branch,
+      "author " + JSON.stringify(this.ctx.author),
+      "message " + JSON.stringify(this.ctx.body),
+    ].join(" ")
+  }
+}
+
+export function addLoad(query: string, poolId) {
+  return query + " | load " + poolId
+}

apps/zui/src/domain/loads/types.ts

@@ -1,20 +1,33 @@
 import {LoadFormat} from "@brimdata/zed-js"
 import {LoadContext} from "./load-context"
+import {LoadReference} from "src/js/state/Loads/types"
 
 export type LoadOptions = {
   windowId: string
   lakeId: string
   poolId: string
-  branch: string
+  branch?: string
   files: string[]
-  shaper: string
+  query?: string
+  shaper?: string
   format?: LoadFormat
   author: string
   body: string
 }
 
 export interface Loader {
-  when(context: LoadContext): PromiseLike<boolean> | boolean
-  run(context: LoadContext): PromiseLike<void> | void
-  rollback(context: LoadContext): PromiseLike<void> | void
+  when(): PromiseLike<boolean> | boolean
+  run(): PromiseLike<void> | void
+  rollback?(): PromiseLike<void> | void
+}
+
+export type LoadEvents = {
+  success: (load: LoadReference) => void
+  abort: (load: LoadReference) => void
+  error: (load: LoadReference) => void
+}
+
+export type LoaderRef = {
+  name: string
+  initialize: (ctx: LoadContext) => Loader
 }

apps/zui/src/domain/pools/plugin-api.ts

@@ -6,10 +6,11 @@ import {loads, window} from "src/zui"
 import * as ops from "./operations"
 import {LoadContext} from "src/domain/loads/load-context"
 import {syncPoolOp} from "src/electron/ops/sync-pool-op"
-import {LoadOptions} from "src/core/loader/types"
 import {getMainObject} from "src/core/main"
 import {TypedEmitter} from "src/util/typed-emitter"
 import {call} from "src/util/call"
+import {LoadOptions} from "../loads/types"
+import {debug} from "src/core/log"
 
 type Events = {
   create: (event: {pool: Pool}) => void
@@ -29,7 +30,7 @@ export class PoolsApi extends TypedEmitter<Events> {
   }
 
   async create(name: string, opts: Partial<CreatePoolOpts> = {}) {
-    const id = await ops.create(window.lakeId, name, opts)
+    const id = await ops.create(name, opts)
     return this.get(id)
   }
 
@@ -41,6 +42,7 @@ export class PoolsApi extends TypedEmitter<Events> {
     const main = getMainObject()
     const context = new LoadContext(main, opts)
     const loader = await loads.initialize(context)
+    debug("Using Loader", loader)
     try {
       await context.setup()
       await loader.run()

apps/zui/src/domain/results/handlers/export.ts

@@ -15,7 +15,12 @@ export const exportToPool = createHandler(
     const query = getExportQuery(null)
     try {
       const poolId = await getOrCreatePool(data)
-      const promise = invoke("results.exportToPool", query, poolId)
+      const promise = invoke(
+        "results.exportToPool",
+        query,
+        poolId,
+        globalThis.windowId
+      )
       toast.promise(promise, {
         loading: "Exporting to pool...",
         success: "Export to pool complete",
@@ -58,9 +63,12 @@ export const exportToFile = createHandler(
 export const exportToClipboard = createHandler(
   async (ctx, format: ResponseFormat) => {
     const query = getExportQuery(format)
-    await ctx.invoke("results.copyToClipboard", query, format)
-
-    ctx.toast.success(`Copied ${format} data to clipboard.`)
+    const promise = ctx.invoke("results.copyToClipboard", query, format)
+    ctx.toast.promise(promise, {
+      loading: "Copying...",
+      error: (e) => errorToString(e),
+      success: `Copied ${format} data to clipboard.`,
+    })
   }
 )
 

apps/zui/src/domain/results/operations.ts

@@ -3,10 +3,8 @@ import {ResponseFormat} from "@brimdata/zed-js"
 import fs from "fs"
 import {pipeline} from "stream"
 import util from "util"
-import {lake} from "src/zui"
+import {lake, pools} from "src/zui"
 import {clipboard} from "electron"
-import {addLoad} from "./utils"
-import { debug } from "src/core/log"
 
 const pipe = util.promisify(pipeline)
 
@@ -46,13 +44,18 @@ export const copyToClipboard = createOperation(
 
 export const exportToPool = createOperation(
   "results.exportToPool",
-  async (ctx, query: string, poolId) => {
+  async (ctx, query: string, poolId: string, windowId: string) => {
     if (!poolId) throw new Error("Argument missing: poolId")
     if (!query) throw new Error("Argument missing: query")
-    const loadQuery = addLoad(query, poolId)
-    debug(loadQuery)
-    const res = await lake.query(loadQuery)
-    const result = await res.js()
-    console.log(result)
+    return pools.load({
+      windowId,
+      lakeId: lake.id,
+      poolId,
+      branch: "main",
+      query,
+      files: [],
+      author: "Zui",
+      body: "Export to pool",
+    })
   }
 )

apps/zui/src/domain/results/utils.ts

@@ -9,7 +9,3 @@ export function cutColumns(query: string, names: string[]) {
 export function addFuse(query: string) {
   return query + " | fuse"
 }
-
-export function addLoad(query: string, poolId) {
-  return query + " | load " + poolId
-}

apps/zui/src/js/api/pools/pools-api.ts

@@ -48,7 +48,7 @@ export class PoolsApi extends ApiDomain {
   }
 
   async create(name: string, opts: Partial<CreatePoolOpts> = {}) {
-    const id = await invoke("pools.create", this.lakeId, name, opts)
+    const id = await invoke("pools.create", name, opts)
     await this.sync(id)
     return id
   }

apps/zui/src/js/components/Toaster.tsx

@@ -15,6 +15,9 @@ const Toaster = () => {
         success: {
           duration: 6000,
         },
+        error: {
+          duration: 15_000,
+        },
       }}
     />
   )

apps/zui/src/js/state/Loads/types.ts

@@ -5,6 +5,7 @@ export type LoadReference = {
   poolId: string
   progress: number
   files: string[]
+  query: string
   startedAt: string
   finishedAt: string | null
   abortedAt: string | null