From 3b0be55c5ce0c4e9bf7e8092c1fa6f7469afc36c Mon Sep 17 00:00:00 2001 From: seveibar Date: Fri, 5 Jul 2024 20:55:56 -0700 Subject: [PATCH 1/2] introduce new level schema and handler to migrate off sqlite --- dev-server-api/src/db/level-db.ts | 98 +++++++++++++++++++++++++++++++ dev-server-api/src/db/schema.ts | 64 ++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 dev-server-api/src/db/level-db.ts create mode 100644 dev-server-api/src/db/schema.ts diff --git a/dev-server-api/src/db/level-db.ts b/dev-server-api/src/db/level-db.ts new file mode 100644 index 00000000..7a363141 --- /dev/null +++ b/dev-server-api/src/db/level-db.ts @@ -0,0 +1,98 @@ +import { Level } from "level" +import { z } from "zod" +import { DBSchema, type DBSchemaType } from "./schema" + +// Create a wrapper class for Level with Zod validation +export class ZodLevelDatabase { + private db: Level + + constructor(location: string) { + this.db = new Level(location) + } + + async get( + collection: K, + id: string + ): Promise { + const key = `${collection}:${id}` + const data = await this.db.get(key) + return DBSchema.shape[collection].parse(JSON.parse(data)) as any + } + + async put( + collection: K, + value: DBSchemaType[K] + ): Promise { + const idkey = `${collection}_id` + const valueLoose: any = value + if (!valueLoose[idkey]) { + // generate an id using the "count" key + let count = await this.db.get(`${collection}:count`) + if (!count) count = 1 + ;(value as any)[idkey] = count + await this.db.put(`${collection}:count`, count + 1) + } + const key = `${collection}:${valueLoose[idkey]}` + const validatedData = DBSchema.shape[collection].parse(value) + await this.db.put(key, JSON.stringify(validatedData)) + } + + async del( + collection: K, + id: string + ): Promise { + const key = `${collection}:${id}` + await this.db.del(key) + } + + async find( + collection: K, + partialObject: Partial + ): Promise { + const results: DBSchemaType[K][] = [] + const schema = DBSchema.shape[collection] + + for await (const [key, value] of this.db.iterator({ + gte: `${collection}:`, + lte: `${collection}:\uffff`, + })) { + try { + const parsedValue = schema.parse(JSON.parse(value)) + if (this.matchesPartialObject(parsedValue, partialObject)) { + return parsedValue as any + } + } catch (error) { + console.error(`Error parsing value for key ${key}:`, error) + } + } + + return null + } + + async findOrThrow( + collection: K, + partialObject: Partial + ): Promise { + const result = await this.find(collection, partialObject) + if (!result) { + throw new Error( + `No record in "${collection}" matches query ${JSON.stringify( + partialObject + )}` + ) + } + return result + } + + private matchesPartialObject( + fullObject: T, + partialObject: Partial + ): boolean { + for (const [key, value] of Object.entries(partialObject)) { + if (fullObject[key as keyof T] !== value) { + return false + } + } + return true + } +} diff --git a/dev-server-api/src/db/schema.ts b/dev-server-api/src/db/schema.ts new file mode 100644 index 00000000..f0079d3e --- /dev/null +++ b/dev-server-api/src/db/schema.ts @@ -0,0 +1,64 @@ +import { z } from "zod" + +// Helper function for nullable fields +const nullableText = () => z.string().nullable() + +// PackageInfo schema +export const PackageInfoSchema = z.object({ + package_info_id: z.number().int(), + name: z.string(), +}) + +// DevPackageExample schema +export const DevPackageExampleSchema = z.object({ + dev_package_example_id: z.number().int(), + file_path: z.string(), + export_name: nullableText(), + tscircuit_soup: z.any().nullable(), // Using any for JSON type + completed_edit_events: z.any().nullable(), // Using any for JSON type + error: nullableText(), + is_loading: z.boolean(), + soup_last_updated_at: nullableText(), + edit_events_last_updated_at: nullableText(), + edit_events_last_applied_at: nullableText(), + last_updated_at: nullableText(), +}) + +// ExportRequest schema +export const ExportRequestSchema = z.object({ + export_request_id: z.number().int(), + example_file_path: nullableText(), + export_parameters: z.any().nullable(), // Using any for JSON type + export_name: nullableText(), + is_complete: z.boolean(), + has_error: z.boolean(), + error: nullableText(), + created_at: nullableText(), +}) + +// ExportFile schema +export const ExportFileSchema = z.object({ + export_file_id: z.number().int(), + file_name: nullableText(), + file_content: z.instanceof(Buffer).nullable(), // For BLOB type + is_complete: z.boolean(), + export_request_id: z.number().int().nullable(), + created_at: nullableText(), +}) + +// Combined DBSchema +export const DBSchema = z.object({ + package_info: PackageInfoSchema, + dev_package_example: DevPackageExampleSchema, + export_request: ExportRequestSchema, + export_file: ExportFileSchema, +}) + +// TypeScript type inference +export type DBSchemaType = z.infer + +// You can also export individual types if needed +export type PackageInfo = z.infer +export type DevPackageExample = z.infer +export type ExportRequest = z.infer +export type ExportFile = z.infer From eee5ce2dcb43458fc1d9f0580280ab6ae37b16ae Mon Sep 17 00:00:00 2001 From: seveibar Date: Fri, 5 Jul 2024 20:56:31 -0700 Subject: [PATCH 2/2] add level as dependency --- dev-server-api/bun.lockb | Bin 67070 -> 72296 bytes dev-server-api/package.json | 1 + 2 files changed, 1 insertion(+) diff --git a/dev-server-api/bun.lockb b/dev-server-api/bun.lockb index d694e6212b402ea8a63082266da5fd1855677a9a..63511a13e143a02a087deaea321122dff5e1644d 100755 GIT binary patch delta 8883 zcmeHNdt6l2_CIrEfB^?cB?pBBNd#d=UNR_(_$m~&GH;V67LO6ec+LzeD$R(bl6n=l zPtp+5@=+=d_(*&vB?>5Ml9&&$?B*lAl~+^L`(0;dYDn$Z?eqJy{qcLgXRW>W+H0@9 z&SuWpKA3O!uL`@xih#Nw=anwXyye5q>c?Mg=+!Xv#k)6V+&weUb8-IVpN(Y;zw!`q zR1OuXy7B!Eluwh{8Pe2K)M?Ny^5Zxu#~E*dUBFks&fxRl_TZ!hy;`fuNYPvBi~Tvy z4f@2KNt4uCj?<{sYGt2r&M$!DB+xHFJ=vKJCOIGd$o}iSIIaVD490wl;|#_(b9L&p zT#j=HG&h`YVI!Cv;DgK#3`GYrxD+xCJVw3FOItpne^$J3{AR5rw0l8AE77zGBT6Z@`OZ$H*;#-yx2{3+yD8_ScCE9*0Ps19d77S>yOpU7?S8aZ%qBXOt%g8$8#sw zz0NK=$-GY&`m~?h{cev{EhAf&Mb!2=f9kVeMrGV{d};o+mJQX$I2P%&&2+~p*+-0N z868=vR4HDKr5()HxJOBc_2f9LlhF>Yp-vXa+}b-z3ZadLmS@c#O7Z8AV%VCFQ4$BF zMx8y_6C)q<(_r6Qq=s zKoUl0K5mhARK?D@L`l4maf5#~mknv8S&}Iwdm)X0#Ifd%O5QDy6$eH0!va|oXlWqp zB9G>e1hVlUmmpRwkCwcE#TbST&Md)2DOm@JJau3`u1bECoQ+pROIBk2Xu4v-#nX_; zeN4W+Qql=Y*I$rIJ1ZsAAUy}E9V_jn`W9})!{zRQ&2&maY{HBU!L+``PQ zcp)_=LW(3kzdVdJg-44ILlMp56;V7N&c^qNmc$}B6dwncpioNQ#w{(Vd5ry#XtC{R zMDZ<1u{1OBfIb+5HROk}#Xb^82oywK$d3j{=3Fbqg1Sbq@yck43P#KcMPVz6U`@(s z{%Qp45*aNC#$$v&h@OvB$*YyDI5L{AQnIGVeunl)V*(Z8&VVoA03dOMV$1EI!E$r7 z%VKGT^1m?kN60y1Ox7S6f^!064niTO27Lh<2;maS3sA9T z3FLTy>L&tJzy@Ld31*pl9FyTBOZ}f=8Xy^Tyh)JJfk$+=@S@WCXQQ01b1@Vms7t1<0Xo z0QK7rP$4F{0-*Xz3-16^A*T9W0Bs_N0cux`@r6>2D-BQs&`jz9npp!tg_t^gK>~^` zll~+?d=Fqmf5g=98-Vy*3tt9P zc@i6JE2!glLUS{d=iggoVyeGl;j0!s5rvdMh4_EiqYeKbo3!BAU)!Tq=FOXC`qww< zzxQZ_@6oN;@@zd?f!3A|fA7(M@6qOL_cHy?vI=b#B~P z^-}TmgV^>(+ymDK;MtbNBC^|%X7FD%HRImsXUdEBsMjrhE@NV+o%JdE7CtQ5{zJi{ z%01o>4_!KYuZ#bgSGpg*wea1F#Hl~U&S~dy@ac7al?O_f$MbAoc4tH1K;^j-?fAXZ zZZ4mYz9M(Lcjr0XzPmlJW3_`Q6<|MdRh*s8(4RHMMwLzD6ZI2O^e|fsu-RJW8Nj=kk+O$9tm$W^8 z=_!v>8-|R1HT%w)S5oF|{Lkc~TgrA(+LNQt#@}V>TAATw;MVQy^7kJ)o7z(P-r<2S zEJ-*rZtxqX#=xA2?c$u1{hWrhzt&n?vRmw;mj2`6`}vXYI^7R{qtE!`yKY{oJSncQ z@ouiIcYzh5H*aWeY<9kI|MHF2V*wS(N4_7E>=^b`S@Y2WAH?@|-4?j!s}YeN1@^6} zd&X^;rjMCCKYM@t(xqFjywosu$fQQ*rIQ(QhdjKpywQG0*9ofZi?jZd_iAqkQ@G#H zGaig_zA*D^(_oR#@m$@r>o1J0_xmirK5+ps#%bzFGMu>VAPJhQc6;hvDW^&d~1 z^XZT>m%_%RKKb1uyM|w7bvoHd_sINpU#B1P8|V4lYuOb?vR57+Hij?xdeei&)kS^t zzPlMxJ0K;svs}5u4!o4^J-1 z9eVcmxX!wklm4cH(KdSO0Mj%QhUn zaPPR}zb^k8{Kv!`b6cluCBCkb&;)e6Jl2Sm(0%W zAHJ+=O^tKm@ph_;E3bw0+H>KP@@LQGY%6%VgK5EtuAh3WuR6Kh@rNJYsp-`^ZTd3j zvR7hu70s^=SU!xM%aM&-usG_Sicw+bl2SiZde!8-S9GnDX+^(+Z~FIeK2_A^%We@- znz9f6(KxJh&BDy1VGm~}h-VipX&F&}|LcIpyxqfJw(+ird1R~D$|+KommMc!tOio- zR4J2d<3!6?u2#kFKspC$1q;xr*oJ9Rwon%*TE)&ldNo(d2I%8N@3KO@iaAb~vg?r6 zu)aAewhPjloH$Vl`yNv23@IBmB~DbzR!vbc*-RGI8l|ib z(pKg%Q^o!TDQ{++sGQY6S~&|d%8L`3SZ*HXmya1is$>CgsYE;2EL?Z8Gq~>GxT=%l?aoxuT%~FZ>vsJhrVE@MTplNu1iIW}uE@y8jf=xFI zCW!Er&CbR8T@;@Q6_a(Q=7J9dJquoB6i0>QQz1T>{3ND+_V`fLn_YBvGhJJp(;XJ_ zcqiilbLJErWN(^c3=bZf3n@PVu!vkY5>Tu%{cas6XgF>UnA*_mgs(+c1*SGMWKV$V z_W`7%_nahv$^n4t==T_XErAUKchI7A0OO;VVEB+l=Lp7UETMdgD;bmkK>!U_4Uj=6 zpa!7w89+LGNaaESvhg`UI(luuXDLB|o%-j5g+_aav&AMEtF7?& zAxrc^LN6!Z0rZ}?6QCE>RX{P5RECLUY(Qn_0TeoVFEjy{0eTnQ4Nxd4bQG$+0EK5i zZ~&n3X~v(jgvxGSTX6RtPy#FkmH`Z)m+HlAeWkBx0jsHe&0{v?6d)DQ0F!|?fiyN? zhp%A*?)m{y0DVVN0+E0(@GQ^+@B{pT0N@fpua8Z@1>hpk0?;euIe;Qf5u+Gw1U3OH z0NRHB2@D2?07HRcw5-qJCJC4bybjRUodG~JFc9bq(9#A1-vIP(dmg9(jsdkm9WWYr z0T>B955xk=KnxH9d)!Quoj>;3%FO!o>OOP@R5Z#gFgng1LXiYwiT!V=;5LVkRGIc zfc5bpzIX%|lzkUjzde5KaK~QX(-3Ms__7|G@Vw9p7p*!uVVWjWch7mw*rxJ1i+Rz+ zU_~EAh$*%z!^L1d0J9#Spf*GitUxkZkGQO7CYBoHqxEpjdNd={getHWPq2nGvmUrv zk898;R1u;upp*4X&U&Kr+d$S+JL{Q`&?!^_bJlY`>!}cG!WCiUg7pNidic-<{?iXG zLQMoFge-rY3kLXrPP6^Zr&rcg5Im@-Kw`$&Mnv)^knkL76Z)Aykh{ zLF6P%0i7HI*{-9ZyfKhnJ1X9typPFvK8W=@7RoOT zVj0I|5!Q3T)L&*A?FLppZ&?Ba+In*6dHd}zCP-SRTWSz9dpWCu1rIs9341-|%&B$~ zuaq-wZK$X9K-JUtr}}jvucV?^umYKYJx0zB)LMGEGfABn@1$V}I z0!gD_n?U;%?0lWf(|Yb&7Iu1c$@*WELO_JohPe*)Mo;Sj?Xm0PR|om+k3fSkML3P; zN2?@QZvh<+VNV|qjj*1-)?V^dj%^qBl*JyJTTf#5v~GTCVEDQe!H^L8A4Az&$4Bz_ zLs{+dP=03kk`tkx*5gy-l&hyyBO+RP5hWqaa7__x^a&ZQ27>KrJ&&GoEv0T=hpXMq z?LzSv%G|)ovDv&~Y-nctPQ<+N`v=VYI}Cry%A3ymHHwzC>nVJkG?q11x|N^we5(s& zXBaCwySvzPbz!F-kUK)YzS*%ucKWyXA|Xp5e>djn=abJ$rY>m+?y?Pku-ijd5ZR|z zG|lC~R~8pec$h{s%=Q~7>}kE_y#(!-_Dr%X>s)akVeNqV`UPt;~k)v2{+At^0E zr_&^v1vcwsS3?^OY_*Z)Ay^^7W_o6FPMTV-)@n1g83|b$d3r*UHd7~@a;M~G$rE!l zX~}IRd1jVglbMl_W@Y?N{ap*a+Eu3)y2|xxogNLyC*QWM;8a_MJX{gU5-$5ZW8(-l z`pqpl_CN*mx$523)^D`26<3tl17w+$j{+bR$l{1KpNYjl`A z{ig`?D(G^uvNE-Loms*EvzRo~2ywYSKJd=I~!FM8IMI delta 5737 zcmeHLYj9NM8UD^m7IKno3`i%N4nm8XY;H@k0hXAs1i1(s|us0z#p%?7i9qLQ*{C1np*jCO|>#DnHJppV^c2UKHA-wV%hh7iticF0FQt(F~Pms@V>YaUN<6RD{e<^XLzx%8Wig* zI#OmQdi1PxNgd)f%1|;ShPugA=h8jKaVRzv!MW*i8M1wd*ER_`93&aNa*aAX zaqMU>&SRW{y4eh+x&fJ+0M)}>mTD+Y0rN`q*xrIlm(qk>BW_^GOJ@tV`=ExhTz_() zoEhr1b)vv4U%7L2l_@iFyv7)ehHHoAb$g5&C=NE%#~V-Mi~uITwz_#UF$J^2gS^we`MS4mKnKT;}_T?%%Nt^dYxO&$wRH=)xj=Ef{o*W4bw{&;rfb#+J2!J*M{VN+Du(*m1 zVFgVi({Y@|c)3jN1Jk-Q{l)|20)R$ld$$2Bza5~FS?fNia|ye`IUrP-<{Q?CUW z_bZFnfoWvcZvbL}=K=cd0BB^sFfGgkUxZx%2eKQWks0s`1*|)>{8fPb8bBj6eh;sP*@%z^#K)0EDzD>_V>|0#@yO##?p5kR{X=D?={jN>6nBeT4i z0`_BU?!WF1JrFR{47)VuW~;IMuYg1Q{|W-lF0VfBdh>4l|I?KF<$wP)<;{(D*{5k+ z&bWi=k}=<>)kx)hzifTPF2|r2NlKMp?t0WNORIcZy?g={HPDxzf7;R%esX=ZK-?-<(Oxef}l@ZF3mx|bU~emS}EhI{W5pHU3OOcG?8zioK<$2 z{FqN$CEFhJ%g0dm$9>vrx%+XyEM8!j>neR(vy@c&W#~e?yn(v4V$}HMG*oGgPg^Ii zK&=VFv(~3=kdj)zjIFlI`%uqF;v&EN4Qjz6pSDTff!g|*T{7x?+7_v-!~7n{Y@wc$ zlzPms2D7dAY1`xzsNGP*7W=g4rC~AVSBu#~?UbAb%x@89+u+k)lrN#WpbD1wv{q?e zg89{9woq*{t`YOAx3}$VlxVqq*))7xMCysowvSf7q)AJxlUkaZtM!~riF^9*^4ElP z6!&QxyY}G$(OcU>8C}yZqxR(o)rEhAjHNFC+_(KGU}4R(@Gi4NsTaZYi2w#zWw>^j z=OzD_Ndj2^l2v8{htJ)1tByw^|C@~9y`5JT2JlP8Wfm&+8>=h|jL$IBUdCSzgm<+x zfSvIg!`Ns5UsH}y*5+6=)6}e zmJx3zXm`lnZ(bkFK^;CG`8CJSM&$vxm^eJu4Zw{6KJAqg;00oA@8Rj$V$)~N%X1*rNI+7AOUXGjN=R7#x#XwElb^Ax^ z!CF27b-*Hv>%lJo4Zu!d2f(%61gr&^>t_M(hGziomL^~c&B>aou>l^o`Jtbe{>Dv7s zSC%VFsqjx_)4#2%jvT)07Sy<1nJ#RnDyQr?>d+gVa^R@DZ}>O9J<$^<4@et$9)6jw z!Te!lhg14|R;<77l&a6%`T?i3eCE)5r^vfd>z#4|v^+)J$J~8C#9py0Rh_JD9y4#3 z_s?h}%Qc9jIF%yR$K3kYDbjw-JtX`)zx_$vlAEt_BF?F@DF`GcSiVnSuJ3` zI~^_3vEr(FwOcFH@pVy7kd+@t$-H`52qN7lNa<-7?{>Eo!f>h1dD} zDGJvhhfOCPH~g@J`5}Z<iJKY?TzvP diff --git a/dev-server-api/package.json b/dev-server-api/package.json index f5cb3674..488a40dc 100644 --- a/dev-server-api/package.json +++ b/dev-server-api/package.json @@ -19,6 +19,7 @@ "better-sqlite3": "^11.0.0", "kysely": "^0.27.3", "kysely-bun-sqlite": "^0.3.2", + "level": "^8.0.1", "redaxios": "^0.5.1", "winterspec": "0.0.81", "zod": "^3.22.4"