diff --git a/api/api.go b/api/api.go index 12693345..600b397b 100644 --- a/api/api.go +++ b/api/api.go @@ -168,7 +168,7 @@ func (api *API) Write(request *frames.WriteRequest, in chan frames.Frame) (int, return nFrames, nRows, errors.Wrap(err, msg) } } else { - api.logger.DebugWith("write request with zero rows", "frames", nFrames, "requst", request) + api.logger.DebugWith("write request with zero rows", "frames", nFrames, "request", request) } ingestDuration := time.Since(ingestStartTime) diff --git a/backends/kv/backend.go b/backends/kv/backend.go index d577f0a4..88bce5e4 100644 --- a/backends/kv/backend.go +++ b/backends/kv/backend.go @@ -119,6 +119,10 @@ func (b *Backend) updateItem(request *frames.ExecRequest) error { func (b *Backend) newConnection(session *frames.Session, password string, token string, path string, addSlash bool) (v3io.Container, string, error) { + // Copy the session to avoid populating the session we received with credentials that may get logged later. + newSession := *session + session = &newSession + session = frames.InitSessionDefaults(session, b.framesConfig) containerName, newPath, err := v3ioutils.ProcessPaths(session, path, addSlash) if err != nil { diff --git a/clients/py/dev-requirements.txt b/clients/py/dev-requirements.txt index 7a1888eb..ddc6a0a3 100644 --- a/clients/py/dev-requirements.txt +++ b/clients/py/dev-requirements.txt @@ -1,7 +1,7 @@ flake8>0 ipython>0 numpydoc>0 -pytest~=4.2 +pytest>=4.2,<6 pyyaml>0 sphinx>0 twine~=1.12 diff --git a/clients/py/requirements.txt b/clients/py/requirements.txt index f6a97523..558d77f8 100644 --- a/clients/py/requirements.txt +++ b/clients/py/requirements.txt @@ -1,5 +1,6 @@ googleapis-common-protos>=1.5.3 -grpcio-tools==1.30.0 -grpcio==1.30.0 +# Can be loosened, but grpcio 1.34.0 must not be used as it segfaults (1.34.1 ok). +grpcio-tools>=1.30,<1.33 +grpcio>=1.30,<1.33 pandas>=0.23.4 requests>=2.19.1 diff --git a/cmd/framulate/Dockerfile b/cmd/framulate/Dockerfile index 4ac4fc4d..0518df5e 100644 --- a/cmd/framulate/Dockerfile +++ b/cmd/framulate/Dockerfile @@ -19,7 +19,7 @@ # (don't forget to update the -p accordingly) -FROM golang:1.12-stretch as build +FROM golang:1.14-stretch as build WORKDIR /frames COPY . . diff --git a/column.go b/column.go index 519c344d..98d38ee8 100644 --- a/column.go +++ b/column.go @@ -57,6 +57,8 @@ func (c *colImpl) Len() int { return len(c.msg.Times) case pb.DType_BOOLEAN: return len(c.msg.Bools) + case pb.DType_NULL: + return -1 } // TODO: panic? diff --git a/frame.go b/frame.go index ae287703..23a14a06 100644 --- a/frame.go +++ b/frame.go @@ -173,10 +173,15 @@ func (fr *frameImpl) Labels() map[string]interface{} { // Len is the number of rows func (fr *frameImpl) Len() int { - if len(fr.columns) > 0 { - return fr.columns[0].Len() + // Use length of first index of length > 0, otherwise the first such column. + for _, slice := range [][]Column{fr.indices, fr.columns} { + for _, column := range slice { + len := column.Len() + if len >= 0 { + return len + } + } } - return 0 } diff --git a/go.sum b/go.sum index 298de3eb..9b3d8cb3 100644 --- a/go.sum +++ b/go.sum @@ -67,21 +67,9 @@ github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871 h1:myF4tU/HdFWU1UzMdf16cHRbownzsyvL7VKIHqkrSvo= github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871/go.mod h1:QD2Bo64oyTWzeV8RFehXS0hZEDFgOK99/h2a6ErRu6E= -github.com/v3io/v3io-go v0.1.9 h1:etkrrRmrI++i8sxGfK/+13f41TxIMohYeZHwVUM62vw= -github.com/v3io/v3io-go v0.1.9/go.mod h1:5poBlcjZG5TiexRTYI44PE6tHzZz5Z60w+iS899pWtc= -github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8 h1:Bg3LtR/4cH9mGWTWKGeEZcf7Z88whuf4mufSnj5S4qg= -github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc= github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186 h1:cHzR1AKhoBVVPNBGt8ekQwXI4wzER1KRb3J3IhJaWao= github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc= -github.com/v3io/v3io-tsdb v0.11.3 h1:zH2rVNsPy9CIm0/gwJ7NNyjDD7G2w4XhUygeZqexLaE= -github.com/v3io/v3io-tsdb v0.11.3/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.4 h1:tDw+EdFNregIy6r4fPqtOybkAj4ngUl4WRQNH24Xe0M= -github.com/v3io/v3io-tsdb v0.11.4/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.5/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= -github.com/v3io/v3io-tsdb v0.11.6/go.mod h1:uyk46LJxROChqmlhg5M5Kw7qK1WfJD3hrqNHR6kLjeg= -github.com/v3io/v3io-tsdb v0.11.7/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= -github.com/v3io/v3io-tsdb v0.11.8 h1:g6fvBgdp57zILC1T2yml9qRcCRVtFsXSKZh7kbeGFpc= -github.com/v3io/v3io-tsdb v0.11.8/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= +github.com/v3io/v3io-tsdb v0.12.0 h1:N2H7wAKyXhE/aFQcCcXQkTRRAKf8yXTGOocbvu4GscQ= github.com/v3io/v3io-tsdb v0.12.0/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= diff --git a/test/kv_integration_test.go b/test/kv_integration_test.go index 1bc0cb94..06ed2c8e 100644 --- a/test/kv_integration_test.go +++ b/test/kv_integration_test.go @@ -453,6 +453,105 @@ func (kvSuite *KvTestSuite) TestNullValuesWrite() { kvSuite.Require().NoError(iter.Err(), "error querying items got") } +// IG-19426 +func (kvSuite *KvTestSuite) TestNullColumnWrite() { + table := fmt.Sprintf("kv_test_nulls%d", time.Now().UnixNano()) + + index := []string{"mike", "joe", "jim"} + indexCol := &pb.Column{ + Kind: pb.Column_SLICE, + Name: "idx", + Dtype: pb.DType_STRING, + Strings: index, + } + + columns := []*pb.Column{ + { + Kind: pb.Column_SLICE, + Name: "n1", + Dtype: pb.DType_NULL, + }, + { + Kind: pb.Column_SLICE, + Name: "n2", + Dtype: pb.DType_INTEGER, + Ints: []int64{1, 2, 3}, + }, + } + + nullValues := initializeNullColumns(len(index)) + nullValues[0].NullColumns["n1"] = true + nullValues[1].NullColumns["n1"] = true + nullValues[2].NullColumns["n1"] = true + + frame := frames.NewFrameFromProto(&pb.Frame{ + Columns: columns, + Indices: []*pb.Column{indexCol}, + NullValues: nullValues, + }) + + wreq := &frames.WriteRequest{ + Backend: kvSuite.backendName, + Table: table, + } + + appender, err := kvSuite.client.Write(wreq) + kvSuite.Require().NoError(err) + + err = appender.Add(frame) + kvSuite.Require().NoError(err) + + err = appender.WaitForComplete(3 * time.Second) + kvSuite.Require().NoError(err) + + input := v3io.GetItemsInput{AttributeNames: []string{"__name", "n1", "n2"}} + + iter, err := v3ioutils.NewAsyncItemsCursor( + kvSuite.v3ioContainer, &input, 1, + nil, kvSuite.internalLogger, + 0, []string{table + "/"}, + "", "") + + var numRows int + for iter.Next() { + currentRow := iter.GetItem() + + key, _ := currentRow.GetFieldString("__name") + + if key != ".#schema" { + numRows++ + } + + switch key { + case ".#schema": + continue + case "mike": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + case "joe": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + case "jim": + kvSuite.Require().Nil(currentRow.GetField("n1"), + "item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1")) + kvSuite.Require().NotNil(currentRow.GetField("n2"), + "item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2")) + default: + kvSuite.T().Fatalf("got an unexpected key '%v'", key) + } + } + + kvSuite.Require().Equal(3, numRows) + + kvSuite.Require().NoError(iter.Err(), "error querying items got") +} + func (kvSuite *KvTestSuite) TestNullValuesRead() { table := fmt.Sprintf("kv_test_nulls_read%d", time.Now().UnixNano())