Reorder datatype constructors to match docs

Add more detail to buildInfo[]
Cleanup extra files from repo

q/arrowkdb.q

@@ -15,20 +15,20 @@ dt.uint64:`arrowkdb 2:(`uint64;1);
 dt.float16:`arrowkdb 2:(`float16;1);
 dt.float32:`arrowkdb 2:(`float32;1);
 dt.float64:`arrowkdb 2:(`float64;1);
-dt.utf8:`arrowkdb 2:(`utf8;1);
-dt.large_utf8:`arrowkdb 2:(`large_utf8;1);
-dt.binary:`arrowkdb 2:(`binary;1);
-dt.large_binary:`arrowkdb 2:(`large_binary;1);
 dt.date32:`arrowkdb 2:(`date32;1);
 dt.date64:`arrowkdb 2:(`date64;1);
 dt.month_interval:`arrowkdb 2:(`month_interval;1);
 dt.day_time_interval:`arrowkdb 2:(`day_time_interval;1);
+dt.binary:`arrowkdb 2:(`binary;1);
+dt.utf8:`arrowkdb 2:(`utf8;1);
+dt.large_binary:`arrowkdb 2:(`large_binary;1);
+dt.large_utf8:`arrowkdb 2:(`large_utf8;1);
 // parameterized datatypes:
-dt.fixed_size_binary:`arrowkdb 2:(`fixed_size_binary;1);
-dt.timestamp:`arrowkdb 2:(`timestamp;1);
 dt.time32:`arrowkdb 2:(`time32;1);
 dt.time64:`arrowkdb 2:(`time64;1);
+dt.timestamp:`arrowkdb 2:(`timestamp;1);
 dt.duration:`arrowkdb 2:(`duration;1);
+dt.fixed_size_binary:`arrowkdb 2:(`fixed_size_binary;1);
 dt.decimal128:`arrowkdb 2:(`decimal128;2);
 // nested datatype constructors:
 // from datatypes:

src/ArrayReader.h

@@ -65,6 +65,9 @@ extern "C"
    * @brief Debugging function which converts a kdb list to an arrow array then
    * converts it back again.
    *
+   * Developer use only - Only useful for manual testing, do not expose in
+   * release version of arrowkdb.q since it has no practical use
+   *
    * @param datatype_id The arrow datatype identifier to use for the intemediate
    * arrow array
    * @param array       The kdb list to be written to the intermediate arrow

src/ArrayWriter.cpp

@@ -195,7 +195,7 @@ void PopulateUnionBuilder(std::shared_ptr<arrow::DataType> datatype, K k_array,
   // 'live' child builder for each union value
   K type_ids = kK(k_array)[0];
   if (type_ids->t != KH)
-    throw TypeCheck("union type_id list not KH");
+    throw TypeCheck("union type_id list not 5h");
 
   // Get all the child builders from the parent union builder
   auto union_builder = static_cast<UnionBuilderType*>(builder);

src/ArrowKdb.cpp

@@ -10,10 +10,11 @@
 #include "ArrowKdb.h"
 
 
+// Main is only used for profiling on windows with arrowkdb.exe
 int main(int argc, char* argv[])
 {
   // khp needs to link with: legacy_stdio_definitions.lib;c_static.lib;ws2_32.lib;Iphlpapi.lib
-  //khp((S)"", -1);
+  // khp((S)"", -1);
   K file;
   if (argc > 1)
     file = ks((S)argv[1]);
@@ -39,25 +40,40 @@ int main(int argc, char* argv[])
   return 0;
 }
 
-K getMemoryPoolStats(K unused)
+EXP K buildInfo(K unused)
 {
-  auto pool = arrow::default_memory_pool();
+  auto info = arrow::GetBuildInfo();
 
-  std::cout << "Bytes allocated: " << pool->bytes_allocated() << std::endl;
-  std::cout << "Max memory: " << pool->max_memory() << std::endl;
-  std::cout << "Backend name:" << pool->backend_name() << std::endl;
+  // Not performance critical so just use join
+  K keys = ktn(KS, 0);
+  K values = ktn(0, 0);
 
-  return (K)0;
-}
+  js(&keys, ss((S)"version"));
+  jk(&values, ki(info.version));
 
-EXP K buildInfo(K unused)
-{
-  auto info = arrow::GetBuildInfo();
+  js(&keys, ss((S)"version_string"));
+  jk(&values, ks((S)info.version_string.c_str()));
+
+  js(&keys, ss((S)"full_so_version"));
+  jk(&values, ks((S)info.full_so_version.c_str()));
+
+  js(&keys, ss((S)"compiler_id"));
+  jk(&values, ks((S)info.compiler_id.c_str()));
+
+  js(&keys, ss((S)"compiler_version"));
+  jk(&values, ks((S)info.compiler_version.c_str()));
+
+  js(&keys, ss((S)"compiler_flags"));
+  jk(&values, ks((S)info.compiler_flags.c_str()));
+
+  js(&keys, ss((S)"git_id"));
+  jk(&values, ks((S)info.git_id.c_str()));
+
+  js(&keys, ss((S)"git_description"));
+  jk(&values, ks((S)info.git_description.c_str()));
 
-  K version = ki(info.version);
-  K so_version = ks((S)info.full_so_version.c_str());
-  K git_desc = ks((S)info.git_description.c_str());
-  K compiler_id = ks((S)(info.compiler_id + info.compiler_version).c_str());
+  js(&keys, ss((S)"package_kind"));
+  jk(&values, ks((S)info.package_kind.c_str()));
 
-  return knk(4, version, so_version, git_desc, compiler_id);
+  return xD(keys, values);
 }

src/ArrowKdb.h

@@ -12,20 +12,12 @@
 
 extern "C"
 {
-  /**
-   * @brief Displays the statistics of the arrow memory pool: bytes allocated,
-   * max memory and backend allocator name
-   * 
-   * @param unused 
-   * @return NULL
-  */
-  EXP K getMemoryPoolStats(K unused);
-
   /**
    * @brief Returns build info regarding the in use arrow library
    *
    * @param unused
-   * @return version, so_version, git_desc, compiler
+   * @return Dictionary detailing various Arrow build info including: Arrow
+   * version, shared object version, git description and compiler used.
   */
   EXP K buildInfo(K unused);
 }

src/FieldStore.cpp

@@ -109,9 +109,10 @@ K field(K field_name, K datatype_id)
     return krr((S)"datatype not found");
 
   // Converting between kdb nulls are arrow nulls would incur a massive
-  // performance hit (2-3x worse).  Also, not all kdb types have a null value,
-  // e.g. KB, KG, KS, 0 of KC, 0 of KG, etc.  So don't allow fields to be
-  // created as nullable (other than NA type which is all nulls).
+  // performance hit (up to 10x worse with trival datatypes that could otherwise
+  // be memcpy'ed).  Also, not all kdb types have a null value, e.g. KB, KG, KS,
+  // 0 of KC, 0 of KG, etc.  So don't allow fields to be created as nullable
+  // (other than NA type which is all nulls).
   bool nullable = datatype->id() == arrow::Type::NA;
   return ki(kx::arrowkdb::GetFieldStore()->Add(arrow::field(kx::arrowkdb::GetKdbString(field_name), datatype, nullable)));
 }

src/SchemaStore.cpp

@@ -149,10 +149,11 @@ K inferSchema(K table)
     auto datatype = kx::arrowkdb::GetArrowType(kK(k_array_data)[i]);
     // Construct each arrow field
 
-    // Converting between kdb nulls are arrow nulls would incur a massive
-    // performance hit (2-3x worse).  Also, not all kdb types have a null value,
-    // e.g. KB, KG, KS, 0 of KC, 0 of KG, etc.  So don't allow fields to be
-    // created as nullable (other than NA type which is all nulls).
+  // Converting between kdb nulls are arrow nulls would incur a massive
+  // performance hit (up to 10x worse with trival datatypes that could otherwise
+  // be memcpy'ed).  Also, not all kdb types have a null value, e.g. KB, KG, KS,
+  // 0 of KC, 0 of KG, etc.  So don't allow fields to be created as nullable
+  // (other than NA type which is all nulls).
     bool nullable = datatype->id() == arrow::Type::NA;
     fields.push_back(arrow::field(field_names[i], datatype, nullable));
   }

src/TableData.cpp

@@ -12,7 +12,6 @@
 #include <arrow/ipc/reader.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/io/memory.h>
-#include <arrow/ipc/feather.h>
 
 #include "TableData.h"
 #include "HelperFunctions.h"

tests/test.t

@@ -480,4 +480,10 @@ ipc.parseArrowToTable[serialized]~table
 sc.removeSchema[schema]
 
 
+-1 "\n+----------|| Test utils ||----------+\n";
+
+show util.buildInfo[]
+(type util.buildInfo[])~99h
+
+
 -1 "\n+----------|| Finished testing ||----------+\n";