From c0093ef80f6b73fdb4939ec81fbea348d887adba Mon Sep 17 00:00:00 2001 From: katyak Date: Sun, 11 Jul 2021 11:32:30 +0300 Subject: [PATCH 1/7] passing list (instead of RepeatedScalarContainer) to pandas Series --- clients/py/v3io_frames/pbutils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/py/v3io_frames/pbutils.py b/clients/py/v3io_frames/pbutils.py index e381700e..aa97c28e 100644 --- a/clients/py/v3io_frames/pbutils.py +++ b/clients/py/v3io_frames/pbutils.py @@ -146,16 +146,16 @@ def msg2df(frame, frame_factory, columns=None, do_reorder=True): def col2series(col, index): current_dtype = "" if col.dtype == fpb.BOOLEAN: - data = col.bools + data = list(col.bools) current_dtype = "bool" elif col.dtype == fpb.FLOAT: - data = col.floats + data = list(col.floats) current_dtype = "float" elif col.dtype == fpb.INTEGER: - data = col.ints + data = list(col.ints) current_dtype = "int" elif col.dtype == fpb.STRING: - data = col.strings + data = list(col.strings) current_dtype = "object" elif col.dtype == fpb.TIME: data = [pd.Timestamp(t, unit='ns') for t in col.times] From bbb1a16fe277d60ccb165d9753a22dc3abe23409 Mon Sep 17 00:00:00 2001 From: katyak Date: Mon, 12 Jul 2021 09:22:48 +0300 Subject: [PATCH 2/7] fixing py test version in python dev-requirements --- clients/py/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/py/dev-requirements.txt b/clients/py/dev-requirements.txt index 7a1888eb..ddc6a0a3 100644 --- a/clients/py/dev-requirements.txt +++ b/clients/py/dev-requirements.txt @@ -1,7 +1,7 @@ flake8>0 ipython>0 numpydoc>0 -pytest~=4.2 +pytest>=4.2,<6 pyyaml>0 sphinx>0 twine~=1.12 From b075b2e1f47e93a2e7d04c85324f7bdf7a87aaee Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Wed, 4 Aug 2021 14:38:34 +0300 Subject: [PATCH 3/7] Loosen grpcio requirements. (#610) --- clients/py/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/py/requirements.txt b/clients/py/requirements.txt index f6a97523..6e945eba 100644 --- a/clients/py/requirements.txt +++ b/clients/py/requirements.txt @@ -1,5 +1,6 @@ googleapis-common-protos>=1.5.3 -grpcio-tools==1.30.0 -grpcio==1.30.0 +# Can be loosened, but grpcio 1.34.0 must not be used as it segfaults (1.34.1 ok). +grpcio-tools>=1.30<1.33 +grpcio>=1.30<1.33 pandas>=0.23.4 requests>=2.19.1 From 6adba3e6d132d311f4be62c1f1c4d6815fbf859e Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Thu, 5 Aug 2021 17:49:33 +0300 Subject: [PATCH 4/7] Fix requirements. (#615) --- clients/py/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/py/requirements.txt b/clients/py/requirements.txt index 6e945eba..558d77f8 100644 --- a/clients/py/requirements.txt +++ b/clients/py/requirements.txt @@ -1,6 +1,6 @@ googleapis-common-protos>=1.5.3 # Can be loosened, but grpcio 1.34.0 must not be used as it segfaults (1.34.1 ok). -grpcio-tools>=1.30<1.33 -grpcio>=1.30<1.33 +grpcio-tools>=1.30,<1.33 +grpcio>=1.30,<1.33 pandas>=0.23.4 requests>=2.19.1 From e3b59ab4e5dad0fb87e69f3ad9503b534e072935 Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Thu, 23 Sep 2021 19:05:53 +0300 Subject: [PATCH 5/7] Stop logging credentials. (#616) --- backends/kv/backend.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backends/kv/backend.go b/backends/kv/backend.go index d577f0a4..88bce5e4 100644 --- a/backends/kv/backend.go +++ b/backends/kv/backend.go @@ -119,6 +119,10 @@ func (b *Backend) updateItem(request *frames.ExecRequest) error { func (b *Backend) newConnection(session *frames.Session, password string, token string, path string, addSlash bool) (v3io.Container, string, error) { + // Copy the session to avoid populating the session we received with credentials that may get logged later. + newSession := *session + session = &newSession + session = frames.InitSessionDefaults(session, b.framesConfig) containerName, newPath, err := v3ioutils.ProcessPaths(session, path, addSlash) if err != nil { From c380cf3c509da8f483221bc780502269a4808234 Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Thu, 30 Sep 2021 18:38:54 +0300 Subject: [PATCH 6/7] IG-19426: fix client.write() when first column is all null (#617) * Fix typo * Use first index for dataframe length, not first column. * More resilient solution. * Add integration test for IG-19426. * impi... --- api/api.go | 2 +- column.go | 2 + frame.go | 11 +++-- test/kv_integration_test.go | 99 +++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/api/api.go b/api/api.go index 12693345..600b397b 100644 --- a/api/api.go +++ b/api/api.go @@ -168,7 +168,7 @@ func (api *API) Write(request *frames.WriteRequest, in chan frames.Frame) (int, return nFrames, nRows, errors.Wrap(err, msg) } } else { - api.logger.DebugWith("write request with zero rows", "frames", nFrames, "requst", request) + api.logger.DebugWith("write request with zero rows", "frames", nFrames, "request", request) } ingestDuration := time.Since(ingestStartTime) diff --git a/column.go b/column.go index 519c344d..98d38ee8 100644 --- a/column.go +++ b/column.go @@ -57,6 +57,8 @@ func (c *colImpl) Len() int { return len(c.msg.Times) case pb.DType_BOOLEAN: return len(c.msg.Bools) + case pb.DType_NULL: + return -1 } // TODO: panic? diff --git a/frame.go b/frame.go index ae287703..23a14a06 100644 --- a/frame.go +++ b/frame.go @@ -173,10 +173,15 @@ func (fr *frameImpl) Labels() map[string]interface{} { // Len is the number of rows func (fr *frameImpl) Len() int { - if len(fr.columns) > 0 { - return fr.columns[0].Len() + // Use length of first index of length > 0, otherwise the first such column. + for _, slice := range [][]Column{fr.indices, fr.columns} { + for _, column := range slice { + len := column.Len() + if len >= 0 { + return len + } + } } - return 0 } diff --git a/test/kv_integration_test.go b/test/kv_integration_test.go index 1bc0cb94..06ed2c8e 100644 --- a/test/kv_integration_test.go +++ b/test/kv_integration_test.go @@ -453,6 +453,105 @@ func (kvSuite *KvTestSuite) TestNullValuesWrite() { kvSuite.Require().NoError(iter.Err(), "error querying items got") } +// IG-19426 +func (kvSuite *KvTestSuite) TestNullColumnWrite() { + table := fmt.Sprintf("kv_test_nulls%d", time.Now().UnixNano()) + + index := []string{"mike", "joe", "jim"} + indexCol := &pb.Column{ + Kind: pb.Column_SLICE, + Name: "idx", + Dtype: pb.DType_STRING, + Strings: index, + } + + columns := []*pb.Column{ + { + Kind: pb.Column_SLICE, + Name: "n1", + Dtype: pb.DType_NULL, + }, + { + Kind: pb.Column_SLICE, + Name: "n2", + Dtype: pb.DType_INTEGER, + Ints: []int64{1, 2, 3}, + }, + } + + nullValues := initializeNullColumns(len(index)) + nullValues[0].NullColumns["n1"] = true + nullValues[1].NullColumns["n1"] = true + nullValues[2].NullColumns["n1"] = true + + frame := frames.NewFrameFromProto(&pb.Frame{ + Columns: columns, + Indices: []*pb.Column{indexCol}, + NullValues: nullValues, + }) + + wreq := &frames.WriteRequest{ + Backend: kvSuite.backendName, + Table: table, + } + + appender, err := kvSuite.client.Write(wreq) + kvSuite.Require().NoError(err) + + err = appender.Add(frame) + kvSuite.Require().NoError(err) + + err = appender.WaitForComplete(3 * time.Second) + kvSuite.Require().NoError(err) + + input := v3io.GetItemsInput{AttributeNames: []string{"__name", "n1", "n2"}} + + iter, err := v3ioutils.NewAsyncItemsCursor( + kvSuite.v3ioContainer, &input, 1, + nil, kvSuite.internalLogger, + 0, []string{table + "/"}, + "", "") + + var numRows int + for iter.Next() { + currentRow := iter.GetItem() + + key, _ := currentRow.GetFieldString("__name") + + if key != ".#schema" { + numRows++ + } + + switch key { + case ".#schema": + continue + case "mike": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + case "joe": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + case "jim": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + default: + kvSuite.T().Fatalf("got an unexpected key '%v'", key) + } + } + + kvSuite.Require().Equal(3, numRows) + + kvSuite.Require().NoError(iter.Err(), "error querying items got") +} + func (kvSuite *KvTestSuite) TestNullValuesRead() { table := fmt.Sprintf("kv_test_nulls_read%d", time.Now().UnixNano()) From ed19c359e9bd8f47ffe30bc39a15859b5fa9fb8e Mon Sep 17 00:00:00 2001 From: Dina Nimrodi Date: Thu, 30 Sep 2021 19:51:06 +0300 Subject: [PATCH 7/7] update framulate go version (#618) Co-authored-by: Dina Nimrodi --- cmd/framulate/Dockerfile | 2 +- go.sum | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/cmd/framulate/Dockerfile b/cmd/framulate/Dockerfile index 4ac4fc4d..0518df5e 100644 --- a/cmd/framulate/Dockerfile +++ b/cmd/framulate/Dockerfile @@ -19,7 +19,7 @@ # (don't forget to update the -p accordingly) -FROM golang:1.12-stretch as build +FROM golang:1.14-stretch as build WORKDIR /frames COPY . . diff --git a/go.sum b/go.sum index 298de3eb..9b3d8cb3 100644 --- a/go.sum +++ b/go.sum @@ -67,21 +67,9 @@ github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871 h1:myF4tU/HdFWU1UzMdf16cHRbownzsyvL7VKIHqkrSvo= github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871/go.mod h1:QD2Bo64oyTWzeV8RFehXS0hZEDFgOK99/h2a6ErRu6E= -github.com/v3io/v3io-go v0.1.9 h1:etkrrRmrI++i8sxGfK/+13f41TxIMohYeZHwVUM62vw= -github.com/v3io/v3io-go v0.1.9/go.mod h1:5poBlcjZG5TiexRTYI44PE6tHzZz5Z60w+iS899pWtc= -github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8 h1:Bg3LtR/4cH9mGWTWKGeEZcf7Z88whuf4mufSnj5S4qg= -github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc= github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186 h1:cHzR1AKhoBVVPNBGt8ekQwXI4wzER1KRb3J3IhJaWao= github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc= -github.com/v3io/v3io-tsdb v0.11.3 h1:zH2rVNsPy9CIm0/gwJ7NNyjDD7G2w4XhUygeZqexLaE= -github.com/v3io/v3io-tsdb v0.11.3/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.4 h1:tDw+EdFNregIy6r4fPqtOybkAj4ngUl4WRQNH24Xe0M= -github.com/v3io/v3io-tsdb v0.11.4/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.5/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.6/go.mod h1:uyk46LJxROChqmlhg5M5Kw7qK1WfJD3hrqNHR6kLjeg= -github.com/v3io/v3io-tsdb v0.11.7/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= -github.com/v3io/v3io-tsdb v0.11.8 h1:g6fvBgdp57zILC1T2yml9qRcCRVtFsXSKZh7kbeGFpc= -github.com/v3io/v3io-tsdb v0.11.8/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= +github.com/v3io/v3io-tsdb v0.12.0 h1:N2H7wAKyXhE/aFQcCcXQkTRRAKf8yXTGOocbvu4GscQ= github.com/v3io/v3io-tsdb v0.12.0/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=