Merge branch 'main' into fix-typelength

LibertyDSNP · Jan 11, 2024 · a9e0eff · a9e0eff
2 parents ede2d9b + 0a42955
commit a9e0eff
Show file tree

Hide file tree

Showing 7 changed files with 77 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -329,12 +329,12 @@ var schema = new parquet.ParquetSchema({
 The Parquet hybrid run length and bitpacking encoding allows to compress runs
 of numbers very efficiently. Note that the RLE encoding can only be used in
 combination with the `BOOLEAN`, `INT32` and `INT64` types. The RLE encoding
-requires an additional `bitWidth` parameter that contains the maximum number of
+requires an additional `typeLength` parameter that contains the maximum number of
 bits required to store the largest value of the field.
 
 ``` js
 var schema = new parquet.ParquetSchema({
-  age: { type: 'UINT_32', encoding: 'RLE', bitWidth: 7 },
+  age: { type: 'UINT_32', encoding: 'RLE', typeLength: 7 },
 });
 ```
 

diff --git a/lib/jsonSchema.ts b/lib/jsonSchema.ts
@@ -69,8 +69,9 @@ const fromJsonSchemaArray = (fieldValue: SupportedJSONSchema4, optionalFieldList
             }
             return fields.createListField('UTF8', optionalFieldList);
         case 'integer':
-        case 'number':
             return fields.createListField('INT64', optionalFieldList);
+        case 'number':
+            return fields.createListField('DOUBLE', optionalFieldList);
         case 'boolean':
             return fields.createListField('BOOLEAN', optionalFieldList);
         case 'object':
@@ -96,8 +97,9 @@ const fromJsonSchemaField = (jsonSchema: JSONSchema4) => (fieldName: string, fie
             }
             return fields.createStringField(optional);
         case 'integer':
-        case 'number':
             return fields.createIntField(64, optional);
+        case 'number':
+            return fields.createDoubleField(optional);
         case 'boolean':
             return fields.createBooleanField(optional);
         case 'array':

diff --git a/test/jsonSchema.test.ts b/test/jsonSchema.test.ts
@@ -58,6 +58,7 @@ describe("Json Schema Conversion Test File", async function () {
     "properties": {
       "string_field": { "type": "string" },
       "int_field": { "type": "integer" },
+      "number_field": { "type": "number" },
       "array_field": {
         "type": "array",
         "items": { "type": "string" },
@@ -118,6 +119,7 @@ describe("Json Schema Conversion Test File", async function () {
   const row1 = {
     string_field: 'string value',
     int_field: 10n,
+    number_field: 2.5,
     timestamp_array_field: { list: [{ element: new Date("2023-01-01 GMT") }] },
 
     timestamp_field: new Date("2023-01-01 GMT"),

diff --git a/test/readme-examples.test.ts b/test/readme-examples.test.ts
@@ -0,0 +1,21 @@
+import { expect } from "chai";
+
+import { ParquetSchema } from '../parquet';
+
+describe("Readme Encoding Examples", function () {
+  it("PLAIN should work", function () {
+    const ps = new ParquetSchema({
+      name: { type: 'UTF8', encoding: 'PLAIN' },
+    });
+    expect(ps).to.be.a("object");
+    expect(ps.schema.name.encoding).to.eq("PLAIN");
+  });
+
+  it("RLE should work", function () {
+    const ps = new ParquetSchema({
+      age: { type: 'UINT_32', encoding: 'RLE', typeLength: 7 },
+    });
+    expect(ps).to.be.a("object");
+    expect(ps.schema.age.typeLength).to.eq(7);
+  });
+});
diff --git a/test/test-files/array.schema.result.json b/test/test-files/array.schema.result.json
@@ -9,7 +9,7 @@
           "fields": {
             "element": {
               "optional": true,
-              "type": "INT64",
+              "type": "DOUBLE",
               "encoding": "PLAIN",
               "compression": "UNCOMPRESSED"
             }
@@ -61,7 +61,7 @@
           "fields": {
             "element": {
               "name": "element",
-              "primitiveType": "INT64",
+              "primitiveType": "DOUBLE",
               "path": [
                 "numberArray",
                 "list",
@@ -148,7 +148,7 @@
           "fields": {
             "element": {
               "name": "element",
-              "primitiveType": "INT64",
+              "primitiveType": "DOUBLE",
               "path": [
                 "numberArray",
                 "list",
@@ -179,7 +179,7 @@
       "fields": {
         "element": {
           "name": "element",
-          "primitiveType": "INT64",
+          "primitiveType": "DOUBLE",
           "path": [
             "numberArray",
             "list",
@@ -195,7 +195,7 @@
     },
     {
       "name": "element",
-      "primitiveType": "INT64",
+      "primitiveType": "DOUBLE",
       "path": [
         "numberArray",
         "list",

diff --git a/test/test-files/json-schema-test-file.result.json b/test/test-files/json-schema-test-file.result.json
@@ -4,7 +4,7 @@
     "type_length": null,
     "repetition_type": null,
     "name": "root",
-    "num_children": 7,
+    "num_children": 8,
     "converted_type": null,
     "scale": null,
     "precision": null,
@@ -35,6 +35,18 @@
     "field_id": null,
     "logicalType": null
   },
+  {
+    "type": 5,
+    "type_length": null,
+    "repetition_type": 1,
+    "name": "number_field",
+    "num_children": null,
+    "converted_type": null,
+    "scale": null,
+    "precision": null,
+    "field_id": null,
+    "logicalType": null
+  },
   {
     "type": null,
     "type_length": null,

diff --git a/test/test-files/json-schema-test-file.schema.result.json b/test/test-files/json-schema-test-file.schema.result.json
@@ -12,6 +12,12 @@
       "encoding": "PLAIN",
       "compression": "UNCOMPRESSED"
     },
+    "number_field": {
+      "optional": true,
+      "type": "DOUBLE",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED"
+    },
     "array_field": {
       "type": "LIST",
       "optional": true,
@@ -157,6 +163,18 @@
       "rLevelMax": 0,
       "dLevelMax": 1
     },
+    "number_field": {
+      "name": "number_field",
+      "primitiveType": "DOUBLE",
+      "path": [
+        "number_field"
+      ],
+      "repetitionType": "OPTIONAL",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED",
+      "rLevelMax": 0,
+      "dLevelMax": 1
+    },
     "array_field": {
       "name": "array_field",
       "path": [
@@ -501,6 +519,18 @@
       "rLevelMax": 0,
       "dLevelMax": 1
     },
+    {
+      "name": "number_field",
+      "primitiveType": "DOUBLE",
+      "path": [
+        "number_field"
+      ],
+      "repetitionType": "OPTIONAL",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED",
+      "rLevelMax": 0,
+      "dLevelMax": 1
+    },
     {
       "name": "array_field",
       "path": [