Add "OmitDefault" and "ExtraFields" options to CRUD

Column definitions may be marked "OmitDefault" at build time, to indicate that
they should not be fetched by default. REST APIs may set "ExtraFields" in order
to include these columns when desired.

Signed-off-by: Andrew Richardson <[email protected]>

pkg/dbsql/crud.go

@@ -119,6 +119,7 @@ type Column[T Resource] struct {
 	Select        string
 	Immutable     bool // disallow update
 	ReadOnly      bool // disallow insert and update
+	OmitDefault   bool // omit from queries unless requested
 	QueryModifier QueryModifier
 	GetFieldPtr   func(inst T) interface{}
 }
@@ -556,6 +557,15 @@ func (c *CrudBase[T]) getCols() (cols namedColumnList[T]) {
 	return cols
 }
 
+func shouldInclude(col string, cols []string) bool {
+	for _, extra := range cols {
+		if col == extra {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *CrudBase[T]) getReadCols(f *ffapi.FilterInfo) (tableFrom string, cols namedColumnList[T], readCols []string, modifiers []QueryModifier) {
 	cols = c.getCols()
 	newCols := namedColumnList[T]{cols[0] /* first column is always the sequence, and must be */}
@@ -571,8 +581,17 @@ func (c *CrudBase[T]) getReadCols(f *ffapi.FilterInfo) (tableFrom string, cols n
 				}
 			}
 		}
-		cols = newCols
+	} else {
+		for i, col := range cols {
+			if i > 0 /* idx==0 handled above */ &&
+				(!col.OmitDefault ||
+					(f != nil && shouldInclude(col.Name, f.ExtraFields))) {
+				newCols = append(newCols, col)
+			}
+		}
 	}
+	cols = newCols
+
 	tableFrom = c.Table
 	if c.ReadTableAlias != "" {
 		tableFrom = fmt.Sprintf("%s AS %s", c.Table, c.ReadTableAlias)

pkg/dbsql/crud_test.go

@@ -303,6 +303,7 @@ func newExtCollection(db *Database, ns string) *CrudBase[*TestLinkable] {
 			"field1": {
 				Select:        "'constant1'",
 				ReadOnly:      true,
+				OmitDefault:   true,
 				GetFieldPtr:   func(inst *TestLinkable) interface{} { return &inst.Field1 },
 				QueryModifier: func(sb sq.SelectBuilder) sq.SelectBuilder { return sb },
 			},
@@ -1278,5 +1279,11 @@ func TestColumnsExt(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, l1Copy, 1)
 	assert.Equal(t, "linked to C1", l1Copy[0].Description)
+	assert.Empty(t, l1Copy[0].Field1)
+
+	l1Copy, _, err = linkables.GetMany(ctx, fb.Eq("id", l1.ID).ExtraFields("field1"))
+	assert.NoError(t, err)
+	assert.Len(t, l1Copy, 1)
+	assert.Equal(t, "linked to C1", l1Copy[0].Description)
 	assert.Equal(t, "constant1", l1Copy[0].Field1)
 }

pkg/ffapi/filter.go

@@ -49,9 +49,12 @@ type FilterModifiers[T any] interface {
 	// Request a count to be returned on the total number that match the query
 	Count(c bool) T
 
-	// Which fields we require to be returned. Only supported when using CRUD layer on top of underlying DB.
+	// List of fields to be returned. Only supported when using CRUD layer on top of underlying DB. Overrides default fields.
 	// Might allow optimization of the query (in the case of SQL DBs, where it can be combined with GroupBy), or request post-query redaction (in the case of document DBs).
 	RequiredFields(...string) T
+
+	// Extra fields to be returned. Extends default fields. Ignored if RequiredFields() is used.
+	ExtraFields(...string) T
 }
 
 // Filter is the output of the builder
@@ -231,6 +234,7 @@ type SortField struct {
 type FilterInfo struct {
 	GroupBy        []string
 	RequiredFields []string
+	ExtraFields    []string
 	Sort           []*SortField
 	Skip           uint64
 	Limit          uint64
@@ -302,6 +306,9 @@ func (f *FilterInfo) String() string {
 	if len(f.RequiredFields) > 0 {
 		val.WriteString(fmt.Sprintf(" requiredFields=%s", strings.Join(f.RequiredFields, ",")))
 	}
+	if len(f.ExtraFields) > 0 {
+		val.WriteString(fmt.Sprintf(" extraFields=%s", strings.Join(f.ExtraFields, ",")))
+	}
 	if len(f.Sort) > 0 {
 		fields := make([]string, len(f.Sort))
 		for i, s := range f.Sort {
@@ -341,6 +348,7 @@ type filterBuilder struct {
 	sort            []*SortField
 	groupBy         []string
 	requiredFields  []string
+	extraFields     []string
 	skip            uint64
 	limit           uint64
 	count           bool
@@ -459,6 +467,7 @@ func (f *baseFilter) Finalize() (fi *FilterInfo, err error) {
 		Sort:           f.fb.sort,
 		GroupBy:        f.fb.groupBy,
 		RequiredFields: f.fb.requiredFields,
+		ExtraFields:    f.fb.extraFields,
 		Skip:           f.fb.skip,
 		Limit:          f.fb.limit,
 		Count:          f.fb.count,
@@ -515,6 +524,20 @@ func (f *baseFilter) RequiredFields(fields ...string) Filter {
 	return f
 }
 
+func (fb *filterBuilder) ExtraFields(fields ...string) FilterBuilder {
+	for _, field := range fields {
+		if _, ok := fb.queryFields[field]; ok {
+			fb.extraFields = append(fb.extraFields, field)
+		}
+	}
+	return fb
+}
+
+func (f *baseFilter) ExtraFields(fields ...string) Filter {
+	_ = f.fb.ExtraFields(fields...)
+	return f
+}
+
 func (fb *filterBuilder) Skip(skip uint64) FilterBuilder {
 	fb.skip = skip
 	return fb

pkg/ffapi/openapi3.go

@@ -346,6 +346,7 @@ func (sg *SwaggerGen) addFilters(ctx context.Context, route *Route, op *openapi3
 		if sg.options.SupportFieldRedaction {
 			sg.AddParam(ctx, op, "query", "fields", "", "", i18n.APIFilterFieldsDesc, false)
 		}
+		sg.AddParam(ctx, op, "query", "extraFields", "", "", i18n.APIFilterExtraFieldsDesc, false)
 	}
 }
 

pkg/ffapi/restfilter.go

@@ -52,20 +52,16 @@ func (hs *HandlerFactory) getValues(values url.Values, key string) (results []st
 	return results
 }
 
-func (hs *HandlerFactory) buildFilter(req *http.Request, ff QueryFactory) (AndFilter, error) {
-	ctx := req.Context()
-	log.L(ctx).Debugf("Query: %s", req.URL.RawQuery)
-	fb := ff.NewFilterLimit(ctx, hs.DefaultFilterLimit)
+func (hs *HandlerFactory) parseFilters(ctx context.Context, form url.Values, filter AndFilter) error {
+	fb := filter.Builder()
 	possibleFields := fb.Fields()
 	sort.Strings(possibleFields)
-	filter := fb.And()
-	_ = req.ParseForm()
 	for _, field := range possibleFields {
-		values := hs.getValues(req.Form, field)
+		values := hs.getValues(form, field)
 		if len(values) == 1 {
 			_, cond, err := hs.getCondition(ctx, fb, field, values[0])
 			if err != nil {
-				return nil, err
+				return err
 			}
 			filter.Condition(cond)
 		} else if len(values) > 0 {
@@ -75,7 +71,7 @@ func (hs *HandlerFactory) buildFilter(req *http.Request, ff QueryFactory) (AndFi
 			for i, value := range values {
 				mods, cond, err := hs.getCondition(ctx, fb, field, value)
 				if err != nil {
-					return nil, err
+					return err
 				}
 				andCombine = andCombine || mods.andCombine
 				fs[i] = cond
@@ -87,6 +83,18 @@ func (hs *HandlerFactory) buildFilter(req *http.Request, ff QueryFactory) (AndFi
 			}
 		}
 	}
+	return nil
+}
+
+func (hs *HandlerFactory) buildFilter(req *http.Request, ff QueryFactory) (AndFilter, error) {
+	ctx := req.Context()
+	log.L(ctx).Debugf("Query: %s", req.URL.RawQuery)
+	fb := ff.NewFilterLimit(ctx, hs.DefaultFilterLimit)
+	filter := fb.And()
+	_ = req.ParseForm()
+	if err := hs.parseFilters(ctx, req.Form, filter); err != nil {
+		return nil, err
+	}
 	skipVals := hs.getValues(req.Form, "skip")
 	if len(skipVals) > 0 {
 		s, _ := strconv.ParseUint(skipVals[0], 10, 64)
@@ -125,6 +133,16 @@ func (hs *HandlerFactory) buildFilter(req *http.Request, ff QueryFactory) (AndFi
 			}
 		}
 	}
+	extraFieldVals := hs.getValues(req.Form, "extraFields")
+	for _, ef := range extraFieldVals {
+		subExtraFieldVals := strings.Split(ef, ",")
+		for _, sef := range subExtraFieldVals {
+			sef = strings.TrimSpace(sef)
+			if sef != "" {
+				filter.ExtraFields(sef)
+			}
+		}
+	}
 	descendingVals := hs.getValues(req.Form, "descending")
 	ascendingVals := hs.getValues(req.Form, "ascending")
 	if len(descendingVals) > 0 && (descendingVals[0] == "" || strings.EqualFold(descendingVals[0], "true")) {

pkg/ffapi/restfilter_test.go

@@ -164,3 +164,17 @@ func TestBuildFilterRequiredFields(t *testing.T) {
 
 	assert.Equal(t, "( created == 0 ) requiredFields=tag,sequence", fi.String())
 }
+
+func TestBuildFilterExtraFields(t *testing.T) {
+	as := &HandlerFactory{
+		MaxFilterLimit: 250,
+	}
+
+	req := httptest.NewRequest("GET", "/things?created=0&extraFields=tag,sequence", nil)
+	filter, err := as.buildFilter(req, TestQueryFactory)
+	assert.NoError(t, err)
+	fi, err := filter.Finalize()
+	assert.NoError(t, err)
+
+	assert.Equal(t, "( created == 0 ) extraFields=tag,sequence", fi.String())
+}

pkg/i18n/en_base_messages.go

@@ -25,16 +25,17 @@ var ffm = func(key, translation string) MessageKey {
 }
 
 var (
-	APISuccessResponse      = ffm("api.success", "Success")
-	APIRequestTimeoutDesc   = ffm("api.requestTimeout", "Server-side request timeout (milliseconds, or set a custom suffix like 10s)")
-	APIFilterParamDesc      = ffm("api.filterParam", "Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^")
-	APIFilterSortDesc       = ffm("api.filterSort", "Sort field. For multi-field sort use comma separated values (or multiple query values) with '-' prefix for descending")
-	APIFilterAscendingDesc  = ffm("api.filterAscending", "Ascending sort order (overrides all fields in a multi-field sort)")
-	APIFilterDescendingDesc = ffm("api.filterDescending", "Descending sort order (overrides all fields in a multi-field sort)")
-	APIFilterSkipDesc       = ffm("api.filterSkip", "The number of records to skip (max: %d). Unsuitable for bulk operations")
-	APIFilterLimitDesc      = ffm("api.filterLimit", "The maximum number of records to return (max: %d)")
-	APIFilterCountDesc      = ffm("api.filterCount", "Return a total count as well as items (adds extra database processing)")
-	APIFilterFieldsDesc     = ffm("api.filterFields", "Comma separated list of fields to return")
+	APISuccessResponse       = ffm("api.success", "Success")
+	APIRequestTimeoutDesc    = ffm("api.requestTimeout", "Server-side request timeout (milliseconds, or set a custom suffix like 10s)")
+	APIFilterParamDesc       = ffm("api.filterParam", "Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^")
+	APIFilterSortDesc        = ffm("api.filterSort", "Sort field. For multi-field sort use comma separated values (or multiple query values) with '-' prefix for descending")
+	APIFilterAscendingDesc   = ffm("api.filterAscending", "Ascending sort order (overrides all fields in a multi-field sort)")
+	APIFilterDescendingDesc  = ffm("api.filterDescending", "Descending sort order (overrides all fields in a multi-field sort)")
+	APIFilterSkipDesc        = ffm("api.filterSkip", "The number of records to skip (max: %d). Unsuitable for bulk operations")
+	APIFilterLimitDesc       = ffm("api.filterLimit", "The maximum number of records to return (max: %d)")
+	APIFilterCountDesc       = ffm("api.filterCount", "Return a total count as well as items (adds extra database processing)")
+	APIFilterFieldsDesc      = ffm("api.filterFields", "Comma separated list of fields to return (replaces the default set of fields)")
+	APIFilterExtraFieldsDesc = ffm("api.filterExtraFields", "Comma separated list of extra fields to return (extends the default set of fields)")
 
 	ResourceBaseID      = ffm("ResourceBase.id", "The UUID of the service")
 	ResourceBaseCreated = ffm("ResourceBase.created", "The time the resource was created")