Merge pull request #105 from m-lab/integration-tests

cloud/bq test coverage improvements

cloud/bq/sanity.go

@@ -110,6 +110,9 @@ func (at *AnnotatedTable) CachedDetail(ctx context.Context) (*Detail, error) {
 	// TODO - use context
 	// TODO - maybe just embed code here.
 	at.detail, at.err = GetTableDetail(ctx, at.dataset, at.Table)
+	if at.err != nil {
+		log.Println(at.FullyQualifiedName(), at.TableID())
+	}
 	return at.detail, at.err
 }
 
@@ -178,6 +181,10 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T
 
 	// TODO - this should take a context?
 	err := dsExt.QueryAndParse(ctx, queryString, &detail)
+	if err != nil {
+		log.Println(err)
+		log.Println("Query error:", queryString)
+	}
 	return &detail, err
 }
 
@@ -239,8 +246,11 @@ func getTableParts(tableName string) (tableNameParts, error) {
 func (at *AnnotatedTable) GetPartitionInfo(ctx context.Context) (*dataset.PartitionInfo, error) {
 	tableName := at.Table.TableID()
 	parts, err := getTableParts(tableName)
-	if err != nil || !parts.isPartitioned {
-		return nil, errors.New("TableID missing partition: " + tableName)
+	if err != nil {
+		return nil, err
+	}
+	if !parts.isPartitioned {
+		return nil, errors.New("TableID does not specify partition: " + tableName)
 	}
 	// Assemble the FQ table name, without the partition suffix.
 	fullTable := fmt.Sprintf("%s:%s.%s", at.ProjectID(), at.DatasetID(), parts.prefix)

cloud/bq/sanity_integration_test.go

@@ -55,7 +55,7 @@ func TestGetTableDetail(t *testing.T) {
 	}
 }
 
-func TestAnnotationTableMeta(t *testing.T) {
+func TestCachedMeta(t *testing.T) {
 	// TODO - Make NewDataSet return a pointer, for consistency with bigquery.
 	ctx := context.Background()
 	dsExt, err := dataset.NewDataset(ctx, "mlab-testing", "src")
@@ -87,25 +87,42 @@ func TestAnnotationTableMeta(t *testing.T) {
 	}
 }
 
-func TestAnnotationDetail(t *testing.T) {
+func TestCachedPartitionInfo(t *testing.T) {
 	ctx := context.Background()
 	dsExt, err := dataset.NewDataset(ctx, "mlab-testing", "src")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	tbl := dsExt.Table("DedupTest")
-	meta, err := tbl.Metadata(ctx)
+	tbl := dsExt.Table("DedupTest$19990101")
+
+	at := bq.NewAnnotatedTable(tbl, &dsExt)
+	// Fetch cache detail - which hits backend
+
+	_, err = at.CachedPartitionInfo(ctx)
 	if err != nil {
 		t.Error(err)
-	} else if meta == nil {
-		t.Error("Meta should not be nil")
 	}
-	at := bq.NewAnnotatedTable(tbl, &dsExt)
-	_, err = at.CachedDetail(ctx)
+	// Fetch again, exercising the cached code path.
+	_, err = at.CachedPartitionInfo(ctx)
 	if err != nil {
 		t.Error(err)
 	}
+
+	badTable := dsExt.Table("non-existant")
+
+	badAT := bq.NewAnnotatedTable(badTable, &dsExt)
+	// Fetch cache detail - which hits backend
+
+	_, err = badAT.CachedPartitionInfo(ctx)
+	if err == nil {
+		t.Error("Should return error")
+	}
+	// Fetch again, exercising the already errored code path.
+	_, err = badAT.CachedPartitionInfo(ctx)
+	if err == nil {
+		t.Error("Should return error")
+	}
 }
 
 func TestGetTablesMatching(t *testing.T) {
@@ -148,4 +165,25 @@ func TestAnnotatedTableGetPartitionInfo(t *testing.T) {
 	} else if info.PartitionID != "" {
 		t.Error("Non-existent partition should return empty PartitionID")
 	}
+
+}
+
+func TestCachedDetail(t *testing.T) {
+	ctx := context.Background()
+	ds, err := dataset.NewDataset(ctx, "mlab-testing", "src")
+	if err != nil {
+		t.Fatal(err)
+	}
+	src := ds.Table("DedupTest")
+	srcAt := bq.NewAnnotatedTable(src, &ds)
+	// Fetch detail.
+	_, err = srcAt.CachedDetail(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Fetch again to exercise cached code path.
+	_, err = srcAt.CachedDetail(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
 }

cloud/bq/sanity_test.go

@@ -5,8 +5,13 @@ import (
 	"log"
 	"strings"
 	"testing"
+	"time"
 
+	"cloud.google.com/go/bigquery"
+	"github.com/GoogleCloudPlatform/google-cloud-go-testing/bigquery/bqiface"
+	"github.com/m-lab/etl-gardener/cloud"
 	"github.com/m-lab/go/dataset"
+	"google.golang.org/api/option"
 )
 
 func init() {
@@ -72,3 +77,67 @@ func TestSanityCheckAndCopy(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+// This defines a Dataset that returns a Table, that returns a canned Metadata.
+type testTable struct {
+	bqiface.Table
+}
+
+func (tbl testTable) Metadata(ctx context.Context) (*bigquery.TableMetadata, error) {
+	meta := bigquery.TableMetadata{CreationTime: time.Now(), LastModifiedTime: time.Now(), NumBytes: 168, NumRows: 8}
+	meta.TimePartitioning = &bigquery.TimePartitioning{Expiration: 0 * time.Second}
+	return &meta, nil
+}
+
+type testDataset struct {
+	bqiface.Dataset
+}
+
+func (ds *testDataset) Table(name string) bqiface.Table {
+	tt := testTable{ds.Dataset.Table(name)}
+	return tt
+}
+
+// creates a Dataset with a dry run client.
+func newTestDataset(project, ds string) dataset.Dataset {
+	ctx := context.Background()
+	dryRun, _ := cloud.DryRunClient()
+	c, err := bigquery.NewClient(ctx, project, option.WithHTTPClient(dryRun))
+	if err != nil {
+		panic(err)
+	}
+
+	bqClient := bqiface.AdaptClient(c)
+
+	return dataset.Dataset{Dataset: &testDataset{bqClient.Dataset(ds)}, BqClient: bqClient}
+}
+
+func TestCachedMeta(t *testing.T) {
+	ctx := context.Background()
+	dsExt := newTestDataset("mlab-testing", "etl")
+
+	tbl := dsExt.Table("DedupTest")
+	meta, err := tbl.Metadata(ctx)
+	if err != nil {
+		t.Error(err)
+	} else if meta == nil {
+		t.Error("Meta should not be nil")
+	}
+
+	at := NewAnnotatedTable(tbl, &dsExt)
+	// Fetch cache detail - which hits backend
+	meta, err = at.CachedMeta(ctx)
+	if err != nil {
+		t.Error(err)
+	} else if meta == nil {
+		t.Error("Meta should not be nil")
+	}
+	// Fetch again, exercising the cached code path.
+	meta, err = at.CachedMeta(ctx)
+	if err != nil {
+		t.Error(err)
+	} else if meta == nil {
+		t.Error("Meta should not be nil")
+	}
+
+}