Merge branch 'master' into flip

attr.go

@@ -41,7 +41,7 @@ const (
 
 // AttrStore represents a storage layer for attributes.
 type AttrStore struct {
-	mu   sync.Mutex
+	mu   sync.RWMutex
 	path string
 	db   *bolt.DB
 
@@ -92,8 +92,8 @@ func (s *AttrStore) Close() error {
 
 // Attrs returns a set of attributes by ID.
 func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+	s.mu.RLock()
+	defer s.mu.RUnlock()
 
 	// Check cache for map.
 	if m = s.attrs[id]; m != nil {
@@ -119,6 +119,19 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
 
 // SetAttrs sets attribute values for a given ID.
 func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
+	// Ignore empty maps.
+	if len(m) == 0 {
+		return nil
+	}
+
+	// Check if the attributes already exist under a read-only lock.
+	if attr, err := s.Attrs(id); err != nil {
+		return err
+	} else if attr != nil && mapContains(attr, m) {
+		return nil
+	}
+
+	// Obtain write lock.
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -493,3 +506,14 @@ func (cur *blockCursor) next() (key, value []byte) {
 
 	return key, value
 }
+
+// mapContains returns true if all keys & values of subset are in m.
+func mapContains(m, subset map[string]interface{}) bool {
+	for k, v := range subset {
+		value, ok := m[k]
+		if !ok || value != v {
+			return false
+		}
+	}
+	return true
+}

attr_test.go

@@ -18,6 +18,8 @@ import (
 	"io/ioutil"
 	"os"
 	"reflect"
+	"runtime"
+	"sync"
 	"testing"
 
 	"github.com/pilosa/pilosa"
@@ -143,6 +145,38 @@ func NewAttrStore() *AttrStore {
 	return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
 }
 
+func BenchmarkAttrStore_Duplicate(b *testing.B) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	const n = 5
+	for i := 0; i < n; i++ {
+		if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	// Update attributes with an existing subset.
+	cpuN := runtime.GOMAXPROCS(0)
+	var wg sync.WaitGroup
+	for i := 0; i < cpuN; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < b.N/cpuN; j++ {
+				if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
+					b.Fatal(err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
 // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
 func MustOpenAttrStore() *AttrStore {
 	s := NewAttrStore()

cmd/root.go

@@ -109,6 +109,11 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
 	v.AutomaticEnv()
 
 	c := v.GetString("config")
+	var flagErr error
+	validTags := make(map[string]bool)
+	flags.VisitAll(func(f *pflag.Flag) {
+		validTags[f.Name] = true
+	})
 
 	// add config file to viper
 	if c != "" {
@@ -118,10 +123,16 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
 		if err != nil {
 			return fmt.Errorf("error reading configuration file '%s': %v", c, err)
 		}
+
+		for _, key := range v.AllKeys() {
+			if _, ok := validTags[key]; !ok {
+				return fmt.Errorf("invalid option in configuration file: %v", key)
+			}
+		}
+
 	}
 
 	// set all values from viper
-	var flagErr error
 	flags.VisitAll(func(f *pflag.Flag) {
 		if flagErr != nil {
 			return

cmd/root_test.go

@@ -171,3 +171,27 @@ func TestRootCommand(t *testing.T) {
 		t.Fatalf("Expected standard usage message from RootCommand, but err: '%v', output: '%s'", err, outStr)
 	}
 }
+
+func TestRootCommand_Config(t *testing.T) {
+	file, err := ioutil.TempFile("", "test.conf")
+	if err != nil {
+		panic(err)
+	}
+	config := `data-dir = "/tmp/pil5_0"
+bind = "127.0.0.1:10101"
+
+[cluster]
+  poll-interval = "2m0s"
+  replicas = 2
+  partitions = 128
+  hosts = [
+    "127.0.0.1:10101",
+    "127.0.0.1:10111",
+  ]`
+	file.Write([]byte(config))
+	file.Close()
+	_, err = ExecNewRootCommand(t, "server", "--config", file.Name())
+	if err.Error() != "invalid option in configuration file: cluster.partitions" {
+		t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err)
+	}
+}

ctl/export.go

@@ -31,7 +31,7 @@ type ExportCommand struct {
 	// Name of the index & frame to export from.
 	Index string
 	Frame string
-	View string
+	View  string
 	// Filename to export to.
 	Path string
 

fragment.go

@@ -86,6 +86,9 @@ type Fragment struct {
 	cache     Cache
 	CacheSize uint32
 
+	// Stats reporting.
+	maxRowID uint64
+
 	// Cache containing full rows (not just counts).
 	rowCache BitmapCache
 
@@ -166,6 +169,11 @@ func (f *Fragment) Open() error {
 		// Clear checksums.
 		f.checksums = make(map[int][]byte)
 
+		// Read last bit to determine max row.
+		pos := f.storage.Max()
+		f.maxRowID = pos / SliceWidth
+		f.stats.Gauge("rows", float64(f.maxRowID))
+
 		return nil
 	}(); err != nil {
 		f.close()
@@ -409,6 +417,12 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
 
 	f.stats.Count("setN", 1)
 
+	// Update row count if they have increased.
+	if rowID > f.maxRowID {
+		f.maxRowID = rowID
+		f.stats.Gauge("rows", float64(f.maxRowID))
+	}
+
 	return changed, nil
 }
 

handler.go

@@ -42,6 +42,7 @@ import (
 
 	_ "github.com/pilosa/pilosa/statik"
 	"github.com/rakyll/statik/fs"
+	"unicode"
 )
 
 // Handler represents an HTTP handler.
@@ -843,6 +844,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro
 // readURLQueryRequest parses query parameters from URL parameters from r.
 func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
 	q := r.URL.Query()
+	validQuery := validOptions(QueryRequest{})
+	for key, _ := range q {
+		if _, ok := validQuery[key]; !ok {
+			return nil, errors.New("invalid query params")
+		}
+	}
 
 	// Parse query string.
 	buf, err := ioutil.ReadAll(r.Body)
@@ -875,6 +882,21 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
 	}, nil
 }
 
+// validOptions return all attributes of an interface with lower first character.
+func validOptions(v interface{}) map[string]bool {
+	validQuery := make(map[string]bool)
+	argsType := reflect.ValueOf(v).Type()
+
+	for i := 0; i < argsType.NumField(); i++ {
+		fieldName := argsType.Field(i).Name
+		chars := []rune(fieldName)
+		chars[0] = unicode.ToLower(chars[0])
+		fieldName = string(chars)
+		validQuery[fieldName] = true
+	}
+	return validQuery
+}
+
 // writeQueryResponse writes the response from the executor to w.
 func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error {
 	if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {

handler_test.go

@@ -209,6 +209,16 @@ func TestHandler_Query_Args_Err(t *testing.T) {
 		t.Fatalf("unexpected body: %q", body)
 	}
 }
+func TestHandler_Query_Params_Err(t *testing.T) {
+	w := httptest.NewRecorder()
+	NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("unexpected status code: %d", w.Code)
+	} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
+		t.Fatalf("unexpected body: %q", body)
+	}
+
+}
 
 // Ensure the handler can execute a query with a uint64 response as JSON.
 func TestHandler_Query_Uint64_JSON(t *testing.T) {

pilosa.go

@@ -49,10 +49,10 @@ var (
 )
 
 // Regular expression to validate index and frame names.
-var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,64}$`)
+var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
 
 // Regular expression to validate row and column labels.
-var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,64}$`)
+var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`)
 
 // ColumnAttrSet represents a set of attributes for a vertical column in an index.
 // Can have a set of attributes attached to it.

pilosa_test.go

@@ -0,0 +1,55 @@
+package pilosa_test
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa"
+)
+
+func TestValidateName(t *testing.T) {
+	names := []string{
+		"a", "ab", "ab1", "b-c", "d_e",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, name := range names {
+		if pilosa.ValidateName(name) != nil {
+			t.Fatalf("Should be valid index name: %s", name)
+		}
+	}
+}
+
+func TestValidateNameInvalid(t *testing.T) {
+	names := []string{
+		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, name := range names {
+		if pilosa.ValidateName(name) == nil {
+			t.Fatalf("Should be invalid index name: %s", name)
+		}
+	}
+}
+
+func TestValidateLabel(t *testing.T) {
+	labels := []string{
+		"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, label := range labels {
+		if pilosa.ValidateLabel(label) != nil {
+			t.Fatalf("Should be valid label: %s", label)
+		}
+	}
+}
+
+func TestValidateLabelInvalid(t *testing.T) {
+	labels := []string{
+		"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, label := range labels {
+		if pilosa.ValidateLabel(label) == nil {
+			t.Fatalf("Should be invalid label: %s", label)
+		}
+	}
+}

pql/ast.go

@@ -72,7 +72,7 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
 	case uint64:
 		return tval, true, nil
 	default:
-		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Calll.UintArg", tval, tval)
+		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval)
 	}
 }