From 53778f4a6b6afe61902bd5d7237a3abbe3ff73f9 Mon Sep 17 00:00:00 2001 From: Sean Boyle Date: Thu, 17 Nov 2022 16:39:25 +0000 Subject: [PATCH 1/2] Initial commit of changes made --- .gitignore.txt | 1 + gitSectionA/dbmaint.q | 149 +++++++++++++++++++++++++++++++++++ gitSectionA/quote.txt | 71 +++++++++++++++++ gitSectionA/sym.q | 7 ++ gitSectionA/trade.txt | 176 ++++++++++++++++++++++++++++++++++++++++++ gitSectionA/util.q | 57 ++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 .gitignore.txt create mode 100644 gitSectionA/dbmaint.q create mode 100644 gitSectionA/quote.txt create mode 100644 gitSectionA/sym.q create mode 100644 gitSectionA/trade.txt create mode 100644 gitSectionA/util.q diff --git a/.gitignore.txt b/.gitignore.txt new file mode 100644 index 0000000..314f02b --- /dev/null +++ b/.gitignore.txt @@ -0,0 +1 @@ +*.txt \ No newline at end of file diff --git a/gitSectionA/dbmaint.q b/gitSectionA/dbmaint.q new file mode 100644 index 0000000..d00d97c --- /dev/null +++ b/gitSectionA/dbmaint.q @@ -0,0 +1,149 @@ +/ kdb+ partitioned database maintenance +\d .os +WIN:.z.o in`w32`w64 +pth:{p:$[10h=type x;x;string x];if[WIN;p[where"/"=p]:"\\"];(":"=first p)_ p} +cpy:{system$[WIN;"copy /v /z ";"cp "],pth[x]," ",pth y} +del:{system$[WIN;"del ";"rm "],pth x} +ren:{system$[WIN;"move ";"mv "],pth[x]," ",pth y} +here:{hsym`$system$[WIN;"cd";"pwd"]} +\d . + +add1col:{[tabledir;colname;defaultvalue] + if[not colname in ac:allcols tabledir; + stdout"adding column ",(string colname)," (type ",(string type defaultvalue),") to `",string tabledir; + num:count get(`)sv tabledir,first ac; + .[(`)sv tabledir,colname;();:;num#defaultvalue]; + @[tabledir;`.d;,;colname]]} + +allcols:{[tabledir]get tabledir,`.d} + +allpaths:{[dbdir;table] + files:key dbdir; + if[any files like"par.txt";:raze allpaths[;table]each hsym each`$read0(`)sv dbdir,`par.txt]; + files@:where files like"[0-9]*";(`)sv'dbdir,'files,'table} + +copy1col:{[tabledir;oldcol;newcol] + if[(oldcol in ac)and not newcol in ac:allcols tabledir; + stdout"copying ",(string oldcol)," to ",(string newcol)," in `",string tabledir; + .os.cpy[(`)sv tabledir,oldcol;(`)sv tabledir,newcol];@[tabledir;`.d;,;newcol]]} + +delete1col:{[tabledir;col] + if[col in ac:allcols tabledir; + stdout"deleting column ",(string col)," from `",string tabledir; + .os.del[(`)sv tabledir,col];@[tabledir;`.d;:;ac except col]]} + +/ +enum:{[tabledir;val] + if[not 11=abs type val;:val]; + .[p;();,;u@:iasc u@:where not(u:distinct enlist val)in v:$[type key p:(`)sv tabledir,`sym;get p;0#`]];`sym!(v,u)?val} +\ + +enum:{[tabledir;val]if[not 11=abs type val;:val];.Q.dd[tabledir;`sym]?val} + + +find1col:{[tabledir;col] + $[col in allcols tabledir; + stdout"column ",string[col]," (type ",(string first"i"$read1((`)sv tabledir,col;8;1)),") in `",string tabledir; + stdout"column ",string[col]," *NOT*FOUND* in `",string tabledir]} + +fix1table:{[tabledir;goodpartition;goodpartitioncols] + if[count missing:goodpartitioncols except allcols tabledir; + stdout"fixing table `",string tabledir;{add1col[x;z;0#get y,z]}[tabledir;goodpartition]each missing]} + +fn1col:{[tabledir;col;fn] + if[col in allcols tabledir; + oldattr:-2!oldvalue:get p:tabledir,col; + newattr:-2!newvalue:fn oldvalue; + if[$[not oldattr~newattr;1b;not oldvalue~newvalue]; + stdout"resaving column ",(string col)," (type ",(string type newvalue),") in `",string tabledir; + oldvalue:0;.[(`)sv p;();:;newvalue]]]} + +reordercols0:{[tabledir;neworder] + if[not((count ac)=count neworder)or all neworder in ac:allcols tabledir;'`order]; + stdout"reordering columns in `",string tabledir; + @[tabledir;`.d;:;neworder]} + +rename1col:{[tabledir;oldname;newname] + if[(oldname in ac)and not newname in ac:allcols tabledir; + stdout"renaming ",(string oldname)," to ",(string newname)," in `",string tabledir; + .os.ren[` sv tabledir,oldname;` sv tabledir,newname];@[tabledir;`.d;:;.[ac;where ac=oldname;:;newname]]]} + +ren1table:{[old;new]stdout"renaming ",(string old)," to ",string new;.os.ren[old;new];} + +add1table:{[dbdir;tablename;table] + stdout"adding ",string tablename; + @[tablename;`;:;.Q.en[dbdir]0#table];} + +stdout:{-1 raze[" "sv string`date`second$.z.P]," ",x;} +validcolname:{(not x in `i,.Q.res,key`.q)and x = .Q.id x} + +////////////////////////////////////////////////////////////////////////////////////////////////////////// +// * public + +thisdb:`:. / if functions are to be run within the database instance then use (`:.) as dbdir + +addcol:{[dbdir;table;colname;defaultvalue] / addcol[`:/data/taq;`trade;`noo;0h] + if[not validcolname colname;'(`)sv colname,`invalid.colname]; + add1col[;colname;enum[dbdir;defaultvalue]]each allpaths[dbdir;table];} + +castcol:{[dbdir;table;col;newtype] / castcol[thisdb;`trade;`size;`short] + fncol[dbdir;table;col;newtype$]} + +clearattrcol:{[dbdir;table;col] / clearattr[thisdb;`trade;`sym] + setattrcol[dbdir;table;col;(`)]} + +copycol:{[dbdir;table;oldcol;newcol] / copycol[`:/k4/data/taq;`trade;`size;`size2] + if[not validcolname newcol;'(`)sv newcol,`invalid.newname]; + copy1col[;oldcol;newcol]each allpaths[dbdir;table];} + +deletecol:{[dbdir;table;col] / deletecol[`:/k4/data/taq;`trade;`iz] + delete1col[;col]each allpaths[dbdir;table];} + +findcol:{[dbdir;table;col] / findcol[`:/k4/data/taq;`trade;`iz] + find1col[;col]each allpaths[dbdir;table];} + +/ adds missing columns, but DOESN'T delete extra columns - do that manually +fixtable:{[dbdir;table;goodpartition] / fixtable[`:/k4/data/taq;`trade;`:/data/taq/2005.02.19] + fix1table[;goodpartition;allcols goodpartition]each allpaths[dbdir;table]except goodpartition;} + +fncol:{[dbdir;table;col;fn] / fncol[thisdb;`trade;`price;2*] + fn1col[;col;fn]each allpaths[dbdir;table];} + +listcols:{[dbdir;table] / listcols[`:/k4/data/taq;`trade] + allcols first allpaths[dbdir;table]} + +renamecol:{[dbdir;table;oldname;newname] / renamecol[`:/k4/data/taq;`trade;`woz;`iz] + if[not validcolname newname;'` sv newname,`invalid.newname]; + rename1col[;oldname;newname]each allpaths[dbdir;table];} + +reordercols:{[dbdir;table;neworder] / reordercols[`:/k4/data/taq;`trade;reverse cols trade] + reordercols0[;neworder]each allpaths[dbdir;table];} + +setattrcol:{[dbdir;table;col;newattr] / setattr[thisdb;`trade;`sym;`g] / `s `p `u + fncol[dbdir;table;col;newattr#]} + +addtable:{[dbdir;tablename;table] / addtable[`:.;`trade;([]price...)] + add1table[dbdir;;table]each allpaths[dbdir;tablename];} + +rentable:{[dbdir;old;new] / rentable[`:.;`trade;`transactions] + ren1table'[allpaths[dbdir;old];allpaths[dbdir;new]];} + +\ +test with https://github.com/KxSystems/kdb/blob/master/tq.q (sample taq database) + +if making changes to current database you need to reload (\l .) to make modifications visible + +if the database you've been modifying is a tick database don't forget to adjust the schema (tick/???.q) to reflect your changes to the data + + +addcol[`:.;`trade;`num;10] +addcol[`:.;`trade;`F;`test] +delete1col[`:./2000.10.02/trade;`F] +fixtable[`:.;`trade;`:./2000.10.03/trade] +reordercols[`:.;`quote;except[2 rotate cols quote;`date]] +clearattrcol[`:.;`trade;`sym] +setattrcol[`:.;`trade;`sym;`p] +castcol[`:.;`trade;`time;`second] +renamecol[`:.;`trade;`price;`PRICE] +pxcols:{(y,())renamecol[`:.;x]'z,()] +`PRICE`size renamecol[`:.;`trade]'`p`s diff --git a/gitSectionA/quote.txt b/gitSectionA/quote.txt new file mode 100644 index 0000000..485c016 --- /dev/null +++ b/gitSectionA/quote.txt @@ -0,0 +1,71 @@ +time sym bid ask bsize asize +2019-06-18D03:05:15.716040000 GS.N 4.194429 28.92601 4 5 +2019-06-18D03:05:17.715982000 GS.N 11.53192 48.36199 9 4 +2019-06-18D03:05:19.715528000 MSFT.O 7.836585 7.540665 1 8 +2019-06-18D03:05:20.715186000 MSFT.O 24.88246 30.54409 4 2 +2019-06-18D03:05:22.714657000 IBM.N 12.80329 2.876258 8 7 +2019-06-18D03:05:24.714268000 IBM.N 47.99482 5.434121 7 3 +2019-06-18D03:05:27.703161000 GS.N 21.64267 31.62057 8 2 +2019-06-18D03:05:32.703864000 MSFT.O 24.22364 38.8941 6 6 +2019-06-18D03:05:36.702931000 GS.N 38.3743 21.11112 7 8 +2019-06-18D03:05:42.717138000 IBM.N 39.67756 44.86179 5 0 +2019-06-18D03:05:44.716618000 GS.N 38.85652 31.78591 4 3 +2019-06-18D03:05:45.716420000 MSFT.O 45.41356 32.07488 4 8 +2019-06-18D03:05:48.710737000 MSFT.O 36.2474 28.09518 4 4 +2019-06-18D03:05:50.704024000 IBM.N 4.991336 40.98007 5 7 +2019-06-18D03:05:51.703557000 MSFT.O 15.0702 48.37882 1 2 +2019-06-18D03:05:53.703353000 GS.N 15.57182 43.24631 2 8 +2019-06-18D03:05:56.711853000 BA.N 32.84367 43.62513 6 9 +2019-06-18D03:05:57.711573000 GS.N 18.07572 17.7519 4 8 +2019-06-18D03:05:58.711376000 MSFT.O 16.69403 26.14589 5 4 +2019-06-18D03:05:59.711168000 BA.N 47.58373 17.07496 4 0 +2019-06-18D03:06:01.710647000 BA.N 39.79765 9.314345 0 9 +2019-06-18D03:06:03.703738000 BA.N 9.363172 15.93321 5 1 +2019-06-18D03:06:04.703629000 GS.N 0.1092236 35.46211 7 3 +2019-06-18D03:06:06.703179000 GS.N 19.61625 17.64781 9 0 +2019-06-18D03:06:07.702876000 BA.N 22.59818 46.28021 2 7 +2019-06-18D03:06:09.702319000 BA.N 5.239721 0.8342128 4 1 +2019-06-18D03:06:10.702161000 GS.N 15.04851 0.2208297 6 4 +2019-06-18D03:06:12.717323000 MSFT.O 46.37225 38.16163 4 8 +2019-06-18D03:06:14.716694000 BA.N 1.405337 32.67986 2 1 +2019-06-18D03:06:16.716342000 MSFT.O 24.18711 32.5099 6 1 +2019-06-18D03:06:18.715898000 IBM.N 23.07442 31.69767 1 5 +2019-06-18D03:06:19.715737000 GS.N 36.08687 28.26131 1 4 +2019-06-18D03:06:20.715418000 BA.N 30.32804 14.53779 5 1 +2019-06-18D03:06:23.714658000 GS.N 15.82207 13.62928 2 9 +2019-06-18D03:06:24.714425000 MSFT.O 38.29525 29.69318 0 4 +2019-06-18D03:06:27.713770000 BA.N 31.07077 42.14983 7 4 +2019-06-18D03:06:33.185300000 MSFT.O 12.00385 7.485022 5 2 +2019-06-18D03:06:33.712352000 BA.N 8.324299 10.96574 6 7 +2019-06-18D03:06:36.711239000 IBM.N 18.81573 44.36463 2 3 +2019-06-18D03:06:37.711231000 MSFT.O 31.61489 20.45336 1 3 +2019-06-18D03:06:39.706445000 MSFT.O 4.038664 23.12236 6 9 +2019-06-18D03:06:40.706252000 MSFT.O 24.2475 46.81004 5 6 +2019-06-18D03:06:42.705819000 BA.N 6.383121 9.113048 0 5 +2019-06-18D03:06:43.705597000 BA.N 26.9158 22.50462 3 8 +2019-06-18D03:06:44.705333000 GS.N 11.85489 12.08228 1 2 +2019-06-18D03:06:47.704608000 BA.N 13.23763 12.97067 5 5 +2019-06-18D03:06:49.704167000 IBM.N 43.99847 31.0608 5 7 +2019-06-18D03:06:52.703447000 MSFT.O 39.60351 33.07996 4 5 +2019-06-18D03:06:53.703062000 MSFT.O 6.65758 46.37964 3 4 +2019-06-18D03:06:55.702757000 GS.N 34.01953 36.8386 9 8 +2019-06-18D03:06:57.702259000 IBM.N 23.66585 27.28721 8 6 +2019-06-18D03:07:00.717191000 GS.N 4.775345 35.20805 5 6 +2019-06-18D03:07:01.716814000 BA.N 32.64634 46.93086 0 6 +2019-06-18D03:07:02.716682000 GS.N 15.7824 9.282377 0 9 +2019-06-18D03:07:03.716454000 IBM.N 28.97735 47.92627 5 1 +2019-06-18D03:07:04.716242000 MSFT.O 6.41355 38.08624 3 2 +2019-06-18D03:07:07.715561000 GS.N 1.062656 12.47356 9 8 +2019-06-18D03:07:11.714443000 IBM.N 3.415027 14.99222 2 6 +2019-06-18D03:07:12.714438000 IBM.N 34.22852 31.04312 7 4 +2019-06-18D03:07:15.709614000 MSFT.O 8.238095 28.05404 7 5 +2019-06-18D03:07:16.716735000 GS.N 0.9810431 21.17852 1 5 +2019-06-18D03:07:17.716585000 IBM.N 12.26745 8.909183 9 9 +2019-06-18D03:07:23.715263000 GS.N 39.31552 13.63137 0 1 +2019-06-18D03:07:26.714542000 IBM.N 5.039161 10.74923 1 0 +2019-06-18D03:07:27.714268000 BA.N 38.01613 40.07364 7 7 +2019-06-18D03:07:28.714092000 BA.N 9.187076 21.18061 0 2 +2019-06-18D03:07:32.712777000 GS.N 14.56379 27.17526 6 2 +2019-06-18D03:07:35.711844000 MSFT.O 27.55229 3.059558 2 0 +2019-06-18D03:07:37.711543000 GS.N 12.33477 40.97651 3 6 +2019-06-18D03:07:38.711250000 IBM.N 21.45296 28.87573 9 4 diff --git a/gitSectionA/sym.q b/gitSectionA/sym.q new file mode 100644 index 0000000..be606b7 --- /dev/null +++ b/gitSectionA/sym.q @@ -0,0 +1,7 @@ +// schema.q +// Quote Table Schema +quote:([]time:`timestamp$();sym:`symbol$();bid:`float$();ask:`float$();bsize:`long$();asize:`long$()); +// trade Table Schema +trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$()); +// Aggregation Table Schema +aggTrade:([sym:`symbol$()];time:`timestamp$();maxTPrice:`float$();minTPice:`float$();tVolume:`long$();bestBid:`float$();bestAsk:`float$()); \ No newline at end of file diff --git a/gitSectionA/trade.txt b/gitSectionA/trade.txt new file mode 100644 index 0000000..a442879 --- /dev/null +++ b/gitSectionA/trade.txt @@ -0,0 +1,176 @@ +time sym price size +2019-06-18D03:05:14.702097000 MSFT.O 8.904193 1 +2019-06-18D03:05:16.716154000 GS.N 26.47404 1 +2019-06-18D03:05:18.715592000 IBM.N 42.40783 9 +2019-06-18D03:05:21.714686000 MSFT.O 9.732546 2 +2019-06-18D03:05:23.714154000 MSFT.O 36.39264 1 +2019-06-18D03:05:25.713658000 IBM.N 33.94541 4 +2019-06-18D03:05:26.713695000 MSFT.O 20.23273 5 +2019-06-18D03:05:28.702702000 IBM.N 8.455216 0 +2019-06-18D03:05:29.704613000 MSFT.O 43.08986 9 +2019-06-18D03:05:30.704194000 BA.N 45.66516 3 +2019-06-18D03:05:31.704051000 IBM.N 27.48968 5 +2019-06-18D03:05:33.703633000 MSFT.O 22.8664 4 +2019-06-18D03:05:34.703364000 IBM.N 16.90049 9 +2019-06-18D03:05:35.703204000 IBM.N 45.12892 4 +2019-06-18D03:05:37.702681000 IBM.N 21.16883 9 +2019-06-18D03:05:38.702432000 BA.N 27.59943 7 +2019-06-18D03:05:39.702253000 IBM.N 22.07246 9 +2019-06-18D03:05:40.717607000 BA.N 11.85644 8 +2019-06-18D03:05:41.717245000 GS.N 0.7970141 3 +2019-06-18D03:05:43.716850000 GS.N 20.50957 1 +2019-06-14D02:15:15.940328000 p 13.50938 8 +2019-06-14D02:15:15.940328000 j 31.78591 1 +2019-06-14D02:15:15.940328000 a 38.85652 9 +2019-06-14D02:15:15.940328000 g 4.113437 5 +2019-06-14D02:15:15.940328000 c 25.66009 4 +2019-06-14D02:15:15.940328000 g 24.73914 6 +2019-06-14D02:15:15.940328000 i 43.32783 6 +2019-06-14D02:15:15.940328000 n 32.07488 1 +2019-06-14D02:15:15.940328000 n 45.41356 8 +2019-06-14D02:15:15.940328000 a 48.98047 5 +2019-06-14D02:15:15.940328000 j 15.38745 4 +2019-06-14D02:15:15.940328000 f 18.26137 9 +2019-06-14D02:15:15.940328000 p 47.95588 2 +2019-06-14D02:15:15.940328000 b 20.60933 7 +2019-06-14D02:15:15.940328000 k 31.65205 0 +2019-06-14D02:15:15.940328000 p 28.76346 1 +2019-06-14D02:15:15.940328000 k 0.4505872 9 +2019-06-14D02:15:15.940328000 o 7.240285 2 +2019-06-14D02:15:15.940328000 o 38.53066 1 +2019-06-14D02:15:15.940328000 m 18.48057 8 +2019-06-14D02:15:15.940328000 f 5.876775 8 +2019-06-14D02:15:15.940328000 a 46.17193 1 +2019-06-14D02:15:15.940328000 o 20.07968 7 +2019-06-14D02:15:15.940328000 n 28.09518 2 +2019-06-14D02:15:15.940328000 c 36.2474 4 +2019-06-14D02:15:15.940328000 d 40.56013 5 +2019-06-14D02:15:15.940328000 o 10.43307 4 +2019-06-14D02:15:15.940328000 m 49.53558 2 +2019-06-14D02:15:15.940328000 h 28.97401 7 +2019-06-14D02:15:15.940328000 m 45.14857 8 +2019-06-14D02:15:15.940328000 n 10.05789 5 +2019-06-14D02:15:15.940328000 e 3.416183 6 +2019-06-14D02:15:15.940328000 m 29.94584 4 +2019-06-14D02:15:15.940328000 m 2.440864 1 +2019-06-14D02:15:15.940328000 h 45.03495 3 +2019-06-14D02:15:15.940328000 o 42.52954 3 +2019-06-14D02:15:15.940328000 h 40.98007 7 +2019-06-14D02:15:15.940328000 k 4.991336 8 +2019-06-14D02:15:15.940328000 a 40.93854 2 +2019-06-14D02:15:15.940328000 j 32.53482 1 +2019-06-14D02:15:15.940328000 b 1.746098 4 +2019-06-14D02:15:15.940328000 l 1.64192 2 +2019-06-14D02:15:15.940328000 d 48.37882 8 +2019-06-14D02:15:15.940328000 d 15.0702 0 +2019-06-14D02:15:15.940328000 h 34.59646 5 +2019-06-14D02:15:15.940328000 c 47.35778 8 +2019-06-14D02:15:15.940328000 o 17.01965 5 +2019-06-14D02:15:15.940328000 o 48.07297 2 +2019-06-14D02:15:15.940328000 c 13.49604 8 +2019-06-14D02:15:15.940328000 p 47.28482 6 +2019-06-14D02:15:15.940328000 m 25.63122 9 +2019-06-14D02:15:15.940328000 b 27.29862 0 +2019-06-14D02:15:15.940328000 l 43.24631 0 +2019-06-14D02:15:15.940328000 h 15.57182 0 +2019-06-14D02:15:15.940328000 d 40.66239 9 +2019-06-14D02:15:15.940328000 b 6.459691 5 +2019-06-14D02:15:15.940328000 e 7.387733 2 +2019-06-14D02:15:15.940328000 a 13.71135 3 +2019-06-14D02:15:15.940328000 m 28.17527 9 +2019-06-14D02:15:15.940328000 n 44.19115 5 +2019-06-14D02:15:15.940328000 g 12.19597 9 +2019-06-14D02:15:15.940328000 e 33.59063 7 +2019-06-14D02:15:15.940328000 g 43.19796 6 +2019-06-14D02:15:15.940328000 h 42.19904 6 +2019-06-14D02:15:15.940328000 m 27.13185 4 +2019-06-14D02:15:15.940328000 k 3.878666 7 +2019-06-14D02:15:15.940328000 m 31.87318 8 +2019-06-14D02:15:15.940328000 g 48.80623 4 +2019-06-14D02:15:15.940328000 l 26.98408 4 +2019-06-14D02:15:15.940328000 d 35.81429 6 +2019-06-14D02:15:15.940328000 m 19.43148 9 +2019-06-14D02:15:15.940328000 i 26.1149 9 +2019-06-14D02:15:15.940328000 e 43.62513 2 +2019-06-14D02:15:15.940328000 i 32.84367 5 +2019-06-14D02:15:15.940328000 a 48.12578 4 +2019-06-14D02:15:15.940328000 o 18.57486 2 +2019-06-14D02:15:15.940328000 b 8.723295 5 +2019-06-14D02:15:15.940328000 p 29.48601 8 +2019-06-14D02:15:15.940328000 a 47.75451 7 +2019-06-14D02:15:15.940328000 f 30.79257 9 +2019-06-14D02:15:15.940328000 p 35.12275 9 +2019-06-14D02:15:15.940328000 l 17.7519 7 +2019-06-14D02:15:15.940328000 b 18.07572 7 +2019-06-14D02:15:15.940328000 c 45.44766 1 +2019-06-14D02:15:15.940328000 m 10.31284 9 +2019-06-14D02:15:15.940328000 k 24.09105 1 +2019-06-14D02:15:15.940328000 c 10.32812 0 +2019-06-14D02:15:15.940328000 a 26.14589 8 +2019-06-14D02:15:15.940328000 d 16.69403 8 +2019-06-14D02:15:15.940328000 n 20.73105 3 +2019-06-14D02:15:15.940328000 l 48.62907 1 +2019-06-14D02:15:15.940328000 i 27.11363 0 +2019-06-14D02:15:15.940328000 b 30.58291 5 +2019-06-14D02:15:15.940328000 g 17.07496 1 +2019-06-14D02:15:15.940328000 d 47.58373 0 +2019-06-14D02:15:15.940328000 d 5.847376 0 +2019-06-14D02:15:15.940328000 i 40.79479 1 +2019-06-14D02:15:15.940328000 m 30.4577 7 +2019-06-14D02:15:15.940328000 o 49.15397 6 +2019-06-14D02:15:15.940328000 l 37.71561 3 +2019-06-18D03:05:46.717742000 GS.N 47.95588 8 +2019-06-18D03:05:47.713266000 MSFT.O 0.4505872 1 +2019-06-18D03:05:49.710650000 BA.N 28.97401 2 +2019-06-18D03:05:52.703554000 BA.N 48.07297 3 +2019-06-18D03:05:54.703077000 MSFT.O 12.19597 9 +2019-06-18D03:05:55.702753000 BA.N 27.13185 1 +2019-06-18D03:06:00.710928000 MSFT.O 49.15397 3 +2019-06-18D03:06:02.703905000 MSFT.O 10.90437 0 +2019-06-18D03:06:05.703418000 MSFT.O 26.40556 7 +2019-06-18D03:06:08.702593000 BA.N 29.65088 1 +2019-06-18D03:06:11.717559000 MSFT.O 22.05331 6 +2019-06-18D03:06:13.717063000 IBM.N 23.99586 0 +2019-06-18D03:06:15.716304000 MSFT.O 7.61333 3 +2019-06-18D03:06:17.716138000 MSFT.O 2.082493 9 +2019-06-18D03:06:21.715169000 BA.N 7.306473 7 +2019-06-18D03:06:22.714998000 BA.N 0.9870703 4 +2019-06-18D03:06:25.714256000 MSFT.O 31.49332 5 +2019-06-18D03:06:26.713913000 IBM.N 27.10083 6 +2019-06-18D03:06:28.713531000 IBM.N 46.81514 8 +2019-06-18D03:06:29.713302000 MSFT.O 6.508518 7 +2019-06-18D03:06:30.713024000 BA.N 37.45107 4 +2019-06-18D03:06:34.712134000 BA.N 15.82718 4 +2019-06-18D03:06:35.711867000 IBM.N 45.51846 7 +2019-06-18D03:06:38.711220000 IBM.N 2.520004 8 +2019-06-18D03:06:41.705966000 MSFT.O 22.9865 0 +2019-06-18D03:06:45.705097000 MSFT.O 43.40338 7 +2019-06-18D03:06:46.704875000 BA.N 12.02221 0 +2019-06-18D03:06:48.704409000 IBM.N 15.23636 7 +2019-06-18D03:06:50.703783000 BA.N 39.59896 9 +2019-06-18D03:06:51.703673000 IBM.N 42.38193 9 +2019-06-18D03:06:54.702846000 GS.N 44.17046 7 +2019-06-18D03:06:56.702364000 IBM.N 47.40359 7 +2019-06-18D03:06:58.717576000 IBM.N 11.81322 0 +2019-06-18D03:06:59.717139000 MSFT.O 18.03648 9 +2019-06-18D03:07:05.715980000 IBM.N 5.988106 8 +2019-06-18D03:07:06.715748000 BA.N 41.31221 2 +2019-06-18D03:07:08.715059000 GS.N 23.20387 8 +2019-06-18D03:07:09.715051000 BA.N 2.192873 5 +2019-06-18D03:07:10.714833000 MSFT.O 12.27739 1 +2019-06-18D03:07:13.717571000 GS.N 40.67242 3 +2019-06-18D03:07:14.709902000 IBM.N 22.5987 5 +2019-06-18D03:07:18.716505000 MSFT.O 9.180919 4 +2019-06-18D03:07:19.716104000 BA.N 31.40824 7 +2019-06-18D03:07:20.715847000 GS.N 45.65717 9 +2019-06-18D03:07:21.715742000 BA.N 37.69352 4 +2019-06-18D03:07:22.715494000 BA.N 8.180249 7 +2019-06-18D03:07:24.715013000 IBM.N 16.49079 9 +2019-06-18D03:07:25.714779000 MSFT.O 49.8727 2 +2019-06-18D03:07:29.713719000 MSFT.O 15.30365 3 +2019-06-18D03:07:30.713292000 GS.N 21.64356 4 +2019-06-18D03:07:31.713123000 MSFT.O 24.97957 9 +2019-06-18D03:07:33.712581000 GS.N 15.32402 9 +2019-06-18D03:07:34.712590000 GS.N 36.21206 8 +2019-06-18D03:07:36.711752000 BA.N 28.83865 9 +2019-06-18D03:07:39.711183000 MSFT.O 28.83055 9 diff --git a/gitSectionA/util.q b/gitSectionA/util.q new file mode 100644 index 0000000..3f24357 --- /dev/null +++ b/gitSectionA/util.q @@ -0,0 +1,57 @@ +////////////////////////////////////////////////////// +// // +// Overwrite of Logging Library // +// // +////////////////////////////////////////////////////// +.log.initialised:0b; +.log.info:{.log.conn@"\n","Info Log @ ",string[.z.Z]," ",("[",";" sv string value .Q.w[]),"]: ",x;}; +.log.error:{.log.conn@"\n","Error Log @ ",string[.z.Z]," ",("[",";" sv string value .Q.w[]),"]: ",x;}; +.log.init:{ + if[.log.initialised=1b;:()]; + .log.lName:$[""~a:getenv`LOGDIR;(("logs/proc"),first string 1+count system "ls logs"),".log";a]; + .log.conn:hopen hsym `$.log.lName; + .log.info"Log Library Initialised: ",.log.lName; + .log.initialiased:1b; + }; +.log.init[]; +////////////////////////////////////////////////////// +// // +// Overwrite of Conn Library // +// // +////////////////////////////////////////////////////// +.conn.priv:1!flip(`cName`h`lastUpdate`status`err)!(()); +.conn.open:{ + `.conn.priv upsert(x;0Ni;.z.Z;`;`); + x1:$[10h=type x;hsym `$x;hsym `$ string x]; + val:@[hopen;x1;{`.conn.priv upsert(x;0Ni;0Nz;`DOWN;`$y);y}[x]]; + $[-6h=type val;[`.conn.priv upsert(x;val;.z.Z;`UP;`);1b];:0b] + }; +.conn.close:{ + `.conn.priv upsert(x;0Ni;.z.Z;`;`); + x1:$[10h=type x;hsym `$x;hsym `$ string x]; + if[not x in key .conn.priv;:.log.error"Invalid Connection Name: Conn Must be present in .conn.priv"]; + `.conn.priv upsert(x;0ni;.z.Z;`CLOSED;`); + hclose x1;1b + }; +.conn.syncSend:{[cName;data] + if[not cName in key .conn.priv;:.log.error"Invalid Connection Name: Conn Must be present in .conn.priv";]; + if[(.conn.priv[cName]`status)in`DOWN`CLOSED;[.log.info"Connection Down! Retrying Connection...";.conn.open[cName]]]; + .[{[cName;data](.conn.priv[cName]`h)(data)};(cName;data); + {$[y like "Cannot write to handle*"; + [`.conn.priv upsert(x;0Ni;.z.Z;`DOWN;`$y);.log.error("Connection Failed: "),string y;]; + x] + }[cName] + ] + }; +.conn.asyncSend:{[cName;data] + if[not cName in key .conn.priv;:.log.error"InValid Connection Name: Conn Must be present in .conn.priv";]; + if[(.conn.priv[cName]`status)in`DOWN`CLOSED;[.log.info"Connection Down! Retrying Connection...";.conn.open[cName]]]; + .[{[cName;data](neg .conn.priv[cName]`h)(data)};(cName;data); + {$[y like "Cannot write to handle*"; + [`.conn.priv upsert(x;0Ni;.z.Z;`DOWN;`$y);0N!raze("Connection Failed: "),string y;]; + x] + }[cName] + ] + }; +.z.ts:{.log.info .Q.s .conn.priv}; +//if[0i=system "t";system "t 60000"]; // if time is not set logging will be created to pub every minute \ No newline at end of file From 81f01327737404e25ae0e826494cd260e768a194 Mon Sep 17 00:00:00 2001 From: Sean Boyle Date: Thu, 17 Nov 2022 16:55:49 +0000 Subject: [PATCH 2/2] Adding new files on scripts branch --- gitSectionB/.DS_Store | Bin 0 -> 6148 bytes gitSectionB/fh.q | 20 ++++++++++++++ gitSectionB/r.q | 59 ++++++++++++++++++++++++++++++++++++++++ gitSectionB/tickNotes.q | 31 +++++++++++++++++++++ gitSectionB/u.q | 47 ++++++++++++++++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 gitSectionB/.DS_Store create mode 100644 gitSectionB/fh.q create mode 100644 gitSectionB/r.q create mode 100644 gitSectionB/tickNotes.q create mode 100644 gitSectionB/u.q diff --git a/gitSectionB/.DS_Store b/gitSectionB/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0.q +\c 20 200 +\l util.q +if[not "w"=first string .z.o;system "sleep 1"]; +.conn.open`localhost:5000; +updVanilla:{if[not x in tables[];:()];.[upsert;(x;flip y);{()}]}; + +computeAggs:{updVanilla[x;y]; + aggTrade:`sym xcols 0!(?[trade;();(enlist`sym)!enlist`sym; + (!). flip + ((`time ;(last;`time)); + (`maxTPrice ;(max;`price)); + (`minTPice ;(min;`price)); + (`tVolume ;(sum;`size)))]) + lj + ?[quote;();(enlist`sym)!enlist`sym; + (!). flip + ((`bestBid ;(max;`bid)); + (`bestAsk ;(min;`ask)))]; + if[0=count aggTrade;:()]; + .conn.asyncSend[`localhost:5000;](`.u.upd;`aggTrade;value flip aggTrade); + }; + .pc.data:(); +defTabs:`trade`quote; +subsTabs:{a:$["," in x;`$"," vs x;`$x];$[1=count a;1#a;a]}getenv `SUBTABS; +// Set Upd Based On Config +upd:$[(getenv `AGGRDB) like "TRUE";computeAggs;updVanilla]; +// Takes Conf Tabs: if AggRDB T&Q Required to complete Aggr +tabs:{$[(getenv `AGGRDB) like "TRUE";defTabs;all subsTabs=`;defTabs;subsTabs]}[]; +/ get the ticker plant and history ports, defaults are 5010,5012 +.u.x:.z.x,(count .z.x)_(":5010";":5012"); +/ end of day: save, clear, hdb reload +.u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[`$":",.u.x 1;`:.;x;`sym];@[;`sym;`g#] each t;}; +/ init schema and sync up from log file +.u.rep:{(.[;();:;].)each x;if[null first y;:()];-11!y; + .log.info "Replaying Log Messages to Ensure Sync Up:","\n",.Q.s tables[]!count each value each tables[]}; +/ connect to ticker plant for (schema;(logcount;log)) +//.u.rep . {schemas:.conn.syncSend[`localhost:5000;](`.u.sub;tabs;`);(enlist schemas),enlist .conn.syncSend[`localhost:5000;](`.u;`i`L)}tabs; +.u.repSpec:{[tpLog;t;sym] + if[b:(a:(get tpLog)[0;0])in system"f";updOld:value a]; + a set {[tc;symc;t;data]if[t<>tc;:()];data:(flip data) where symc=data[1];.[upsert;(t;data);{()}]}[t;sym;;]; + -11!tpLog; + //if[b;a set updOld]; + .log.info"Replayed ",string[t]," Log for Sym: ",string[sym]," - ",string[count[value[t]]]," rows"; + }; + //.u.PubCSVtoTP[`:trade.csv;`localhost:5000] + .u.PubCSVtoTP:{ + data:(exec t from meta value a:-4_except[;":"]string x;enlist",")0:x; + .conn.asyncSend[y;(`upd;`$a;data)] + }; + + + + + + + + diff --git a/gitSectionB/tickNotes.q b/gitSectionB/tickNotes.q new file mode 100644 index 0000000..ca5b66c --- /dev/null +++ b/gitSectionB/tickNotes.q @@ -0,0 +1,31 @@ +// ** tickNotes ** +/ + globals used + .u.w - dictionary of tables->(handle;syms) + .u.i - msg count in log file + .u.j - total msg count (log file plus those held in buffer) + .u.t - table names + .u.L - tp log filename, e.g. `:./sym2008.09.11 + .u.l - handle to tp log file + .u.d - date +\ + +////////////////////////// INSTRUCTIONS TO RUN ////////////////////////// +// TICKERPLANT +q tick.q sym logFiles/ -p 5000 + +// FEEDHANDLER +q tick/fh.q sym 5010 + +// RDB1 - subscribing for quote and trade and publishing aggTrade Table +export AGGRDB="TRUE" +q tick/r.q localhost:5000 localhost:5002 -p 5005 + + +// RDB2 - subscribing for aggTrade Table Only +export SUBTABS="aggTrade" +q tick/r.q localhost:5000 localhost:5002 -p 5006 + +// HDB +q sym -p 5012 +///////////////////////////////////////////////////////////////////////// \ No newline at end of file diff --git a/gitSectionB/u.q b/gitSectionB/u.q new file mode 100644 index 0000000..b5d1428 --- /dev/null +++ b/gitSectionB/u.q @@ -0,0 +1,47 @@ +// u.q +\c 20 200 +\l util.q +.u.subs:([]h:`int$();tabs:();syms:()); +.u.getH:{.z.w in exec h from .u.subs}; +.u.getHByTab:{exec distinct h from .u.subs where x in ' tabs}; +.u.modSub:{[hdl;tab]vals:value exec from .u.subs where h=hdl;`.u.subs upsert@[vals;1;union;tab]}; +.u.getHByTab:{[x;y]exec distinct h from .u.subs where x in ' tabs,y in ' syms}; +.u.sel:{$[`~y;x;select from x where sym in y]}; +////////////////////////////////////////////////// +.u.init:{.u.w:.u.t!(count .u.t:tables`)#()}; +// Adding a New Subscription to the TickerPlant +.u.add:{ + $[not count tabs:.u.t inter x; // Check if any table exists + :.log.error"Error: Invalid Table Name"; + (not .u.getH`); + `.u.subs upsert (.z.w;tabs;enlist y); // Make Sub + .u.modSub[.z.w;tabs]]; // Otherwise Modify Subs + // Output Tab Names and Schema + {(x;$[99=type v:value x;.u.sel[v]y;0#v])}[;y]each x + }; +// Publishing Data to Handles +.u.pub:{[t;x] + if[not count han:.u.getHByTab[t;exec sym from x];:()]; + {[h;tab;data](neg h)(`upd;tab;data)}[;t;x] each han + }; +.u.getHByTab:{[x;y]exec distinct h from .u.subs where x in ' tabs,(`;y) in ':first each syms} +// Deleting Subscriptions from the .u.sub handle +.u.del:{ + delete from `.u.subs where h=x + }; +// Port Close Call Defined +.z.pc:{.u.del[x]}; +// .u.sub call registering subs +.u.sub:{ + if[x~`;:.u.add[.u.t;y]]; + .u.add[x;y] + }; +.u.EndOfDay:{ + t:tables[]except`aggTrade; + upd::{.[upsert;(x;flip y);{()}]};-11!.u.L; + .Q.dpft[`:HDB;.z.D;`sym;]each t; + compCols:{a:("HDB/",string[.z.D],"/"),/:string x;b:string except[;`time]each`$system each "ls ",/:a;raze hsym each '`$(a,'"/"),/:'b}t; + {-19!(x;x;17;2;6)}each compCols; + {delete from x}each tables[]; + }; +.u.end:{{x(`.u.end)}each exec h from .u.subs}; \ No newline at end of file