Merge pull request #619 from v3io/development

dev -> master

api/api.go

@@ -168,7 +168,7 @@ func (api *API) Write(request *frames.WriteRequest, in chan frames.Frame) (int,
 			return nFrames, nRows, errors.Wrap(err, msg)
 		}
 	} else {
-		api.logger.DebugWith("write request with zero rows", "frames", nFrames, "requst", request)
+		api.logger.DebugWith("write request with zero rows", "frames", nFrames, "request", request)
 	}
 
 	ingestDuration := time.Since(ingestStartTime)

backends/kv/backend.go

@@ -119,6 +119,10 @@ func (b *Backend) updateItem(request *frames.ExecRequest) error {
 
 func (b *Backend) newConnection(session *frames.Session, password string, token string, path string, addSlash bool) (v3io.Container, string, error) {
 
+	// Copy the session to avoid populating the session we received with credentials that may get logged later.
+	newSession := *session
+	session = &newSession
+
 	session = frames.InitSessionDefaults(session, b.framesConfig)
 	containerName, newPath, err := v3ioutils.ProcessPaths(session, path, addSlash)
 	if err != nil {

clients/py/dev-requirements.txt

@@ -1,7 +1,7 @@
 flake8>0
 ipython>0
 numpydoc>0
-pytest~=4.2
+pytest>=4.2,<6
 pyyaml>0
 sphinx>0
 twine~=1.12

clients/py/requirements.txt

@@ -1,5 +1,6 @@
 googleapis-common-protos>=1.5.3
-grpcio-tools==1.30.0
-grpcio==1.30.0
+# Can be loosened, but grpcio 1.34.0 must not be used as it segfaults (1.34.1 ok).
+grpcio-tools>=1.30,<1.33
+grpcio>=1.30,<1.33
 pandas>=0.23.4
 requests>=2.19.1

cmd/framulate/Dockerfile

@@ -19,7 +19,7 @@
 # (don't forget to update the -p accordingly)
 
 
-FROM golang:1.12-stretch as build
+FROM golang:1.14-stretch as build
 
 WORKDIR /frames
 COPY . .

column.go

@@ -57,6 +57,8 @@ func (c *colImpl) Len() int {
 		return len(c.msg.Times)
 	case pb.DType_BOOLEAN:
 		return len(c.msg.Bools)
+	case pb.DType_NULL:
+		return -1
 	}
 
 	// TODO: panic?

frame.go

@@ -173,10 +173,15 @@ func (fr *frameImpl) Labels() map[string]interface{} {
 
 // Len is the number of rows
 func (fr *frameImpl) Len() int {
-	if len(fr.columns) > 0 {
-		return fr.columns[0].Len()
+	// Use length of first index of length > 0, otherwise the first such column.
+	for _, slice := range [][]Column{fr.indices, fr.columns} {
+		for _, column := range slice {
+			len := column.Len()
+			if len >= 0 {
+				return len
+			}
+		}
 	}
-
 	return 0
 }
 

go.sum

@@ -67,21 +67,9 @@ github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y=
 github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871 h1:myF4tU/HdFWU1UzMdf16cHRbownzsyvL7VKIHqkrSvo=
 github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871/go.mod h1:QD2Bo64oyTWzeV8RFehXS0hZEDFgOK99/h2a6ErRu6E=
-github.com/v3io/v3io-go v0.1.9 h1:etkrrRmrI++i8sxGfK/+13f41TxIMohYeZHwVUM62vw=
-github.com/v3io/v3io-go v0.1.9/go.mod h1:5poBlcjZG5TiexRTYI44PE6tHzZz5Z60w+iS899pWtc=
-github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8 h1:Bg3LtR/4cH9mGWTWKGeEZcf7Z88whuf4mufSnj5S4qg=
-github.com/v3io/v3io-go v0.2.4-0.20201209122348-e01fabe35df8/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc=
 github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186 h1:cHzR1AKhoBVVPNBGt8ekQwXI4wzER1KRb3J3IhJaWao=
 github.com/v3io/v3io-go v0.2.5-0.20210113095419-6c806b8d5186/go.mod h1:WGxAG5MfZ5FeeZa7yGzocW+iLxTlrpkOWdnCIty4QDc=
-github.com/v3io/v3io-tsdb v0.11.3 h1:zH2rVNsPy9CIm0/gwJ7NNyjDD7G2w4XhUygeZqexLaE=
-github.com/v3io/v3io-tsdb v0.11.3/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548=
-github.com/v3io/v3io-tsdb v0.11.4 h1:tDw+EdFNregIy6r4fPqtOybkAj4ngUl4WRQNH24Xe0M=
-github.com/v3io/v3io-tsdb v0.11.4/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548=
-github.com/v3io/v3io-tsdb v0.11.5/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548=
-github.com/v3io/v3io-tsdb v0.11.6/go.mod h1:uyk46LJxROChqmlhg5M5Kw7qK1WfJD3hrqNHR6kLjeg=
-github.com/v3io/v3io-tsdb v0.11.7/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU=
-github.com/v3io/v3io-tsdb v0.11.8 h1:g6fvBgdp57zILC1T2yml9qRcCRVtFsXSKZh7kbeGFpc=
-github.com/v3io/v3io-tsdb v0.11.8/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU=
+github.com/v3io/v3io-tsdb v0.12.0 h1:N2H7wAKyXhE/aFQcCcXQkTRRAKf8yXTGOocbvu4GscQ=
 github.com/v3io/v3io-tsdb v0.12.0/go.mod h1:zAz77gck9fjlC+lPbbq4JBHitP3rv4KLWR/YUuFjJcU=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=

test/kv_integration_test.go

@@ -453,6 +453,105 @@ func (kvSuite *KvTestSuite) TestNullValuesWrite() {
 	kvSuite.Require().NoError(iter.Err(), "error querying items got")
 }
 
+// IG-19426
+func (kvSuite *KvTestSuite) TestNullColumnWrite() {
+	table := fmt.Sprintf("kv_test_nulls%d", time.Now().UnixNano())
+
+	index := []string{"mike", "joe", "jim"}
+	indexCol := &pb.Column{
+		Kind:    pb.Column_SLICE,
+		Name:    "idx",
+		Dtype:   pb.DType_STRING,
+		Strings: index,
+	}
+
+	columns := []*pb.Column{
+		{
+			Kind:  pb.Column_SLICE,
+			Name:  "n1",
+			Dtype: pb.DType_NULL,
+		},
+		{
+			Kind:  pb.Column_SLICE,
+			Name:  "n2",
+			Dtype: pb.DType_INTEGER,
+			Ints:  []int64{1, 2, 3},
+		},
+	}
+
+	nullValues := initializeNullColumns(len(index))
+	nullValues[0].NullColumns["n1"] = true
+	nullValues[1].NullColumns["n1"] = true
+	nullValues[2].NullColumns["n1"] = true
+
+	frame := frames.NewFrameFromProto(&pb.Frame{
+		Columns:    columns,
+		Indices:    []*pb.Column{indexCol},
+		NullValues: nullValues,
+	})
+
+	wreq := &frames.WriteRequest{
+		Backend: kvSuite.backendName,
+		Table:   table,
+	}
+
+	appender, err := kvSuite.client.Write(wreq)
+	kvSuite.Require().NoError(err)
+
+	err = appender.Add(frame)
+	kvSuite.Require().NoError(err)
+
+	err = appender.WaitForComplete(3 * time.Second)
+	kvSuite.Require().NoError(err)
+
+	input := v3io.GetItemsInput{AttributeNames: []string{"__name", "n1", "n2"}}
+
+	iter, err := v3ioutils.NewAsyncItemsCursor(
+		kvSuite.v3ioContainer, &input, 1,
+		nil, kvSuite.internalLogger,
+		0, []string{table + "/"},
+		"", "")
+
+	var numRows int
+	for iter.Next() {
+		currentRow := iter.GetItem()
+
+		key, _ := currentRow.GetFieldString("__name")
+
+		if key != ".#schema" {
+			numRows++
+		}
+
+		switch key {
+		case ".#schema":
+			continue
+		case "mike":
+			kvSuite.Require().Nil(currentRow.GetField("n1"),
+				"item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1"))
+
+			kvSuite.Require().NotNil(currentRow.GetField("n2"),
+				"item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2"))
+		case "joe":
+			kvSuite.Require().Nil(currentRow.GetField("n1"),
+				"item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1"))
+
+			kvSuite.Require().NotNil(currentRow.GetField("n2"),
+				"item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2"))
+		case "jim":
+			kvSuite.Require().Nil(currentRow.GetField("n1"),
+				"item %v - key n1 supposed to be null but got %v", key, currentRow.GetField("n1"))
+			kvSuite.Require().NotNil(currentRow.GetField("n2"),
+				"item %v - key n2 supposed to be null but got %v", key, currentRow.GetField("n2"))
+		default:
+			kvSuite.T().Fatalf("got an unexpected key '%v'", key)
+		}
+	}
+
+	kvSuite.Require().Equal(3, numRows)
+
+	kvSuite.Require().NoError(iter.Err(), "error querying items got")
+}
+
 func (kvSuite *KvTestSuite) TestNullValuesRead() {
 	table := fmt.Sprintf("kv_test_nulls_read%d", time.Now().UnixNano())