Add Series Option to QB

specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts

@@ -123,6 +123,7 @@ describe('serializeResource', () => {
       remarks: null,
       searchSynonymy: null,
       selectDistinct: null,
+      selectSeries: null,
       smushed: null,
       specifyUser: null,
       sqlStr: null,

specifyweb/frontend/js_src/lib/components/DataModel/types.ts

@@ -5291,6 +5291,7 @@ export type SpQuery = {
     readonly remarks: string | null;
     readonly searchSynonymy: boolean | null;
     readonly selectDistinct: boolean | null;
+    readonly selectSeries: boolean | null;
     readonly smushed: boolean | null;
     readonly sqlStr: string | null;
     readonly timestampCreated: string;

specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx

@@ -105,6 +105,7 @@ overrideAjax(
       resource_uri: undefined,
       searchsynonymy: null,
       selectdistinct: false,
+      selectseries: false,
       smushed: null,
       specifyuser: '/api/specify/specifyuser/2/',
       sqlstr: null,

specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx

@@ -13,6 +13,8 @@ export function QueryToolbar({
   showHiddenFields,
   tableName,
   isDistinct,
+  isSeries,
+  showSeries,
   onToggleHidden: handleToggleHidden,
   onToggleDistinct: handleToggleDistinct,
   onRunCountOnly: handleRunCountOnly,
@@ -21,6 +23,8 @@ export function QueryToolbar({
   readonly showHiddenFields: boolean;
   readonly tableName: keyof Tables;
   readonly isDistinct: boolean;
+  readonly isSeries: boolean;
+  readonly showSeries: boolean;
   readonly onToggleHidden: (value: boolean) => void;
   readonly onToggleDistinct: () => void;
   readonly onRunCountOnly: () => void;
@@ -38,6 +42,15 @@ export function QueryToolbar({
       <span className="-ml-2 flex-1" />
       {hasPermission('/querybuilder/query', 'execute') && (
         <>
+          {showSeries && (
+            <Label.Inline>
+              <Input.Checkbox
+                checked={isSeries}
+                onChange={handleToggleSeries}
+              />
+              {queryText.series()}
+            </Label.Inline>
+          )}
           {/*
            * Query Distinct for trees is disabled because of
            * https://github.com/specify/specify7/pull/1019#issuecomment-973525594

specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx

@@ -94,6 +94,7 @@ function Wrapped({
   readonly onChange?: (props: {
     readonly fields: RA<SerializedResource<SpQueryField>>;
     readonly isDistinct: boolean | null;
+    readonly isSeries: boolean | null;
   }) => void;
 }): JSX.Element {
   const [query, setQuery] = useResource(queryResource);
@@ -157,8 +158,9 @@ function Wrapped({
     handleChange?.({
       fields: unParseQueryFields(state.baseTableName, state.fields),
       isDistinct: query.selectDistinct,
+      isSeries: query.selectSeries,
     });
-  }, [state, query.selectDistinct]);
+  }, [state, query.selectDistinct, query.selectSeries]);
 
   /**
    * If tried to save a query, enforce the field length limit for the
@@ -296,6 +298,10 @@ function Wrapped({
     undefined
   );
 
+  const showSeries =
+    table.name === 'CollectionObject' &&
+    state.fields.some((field) => field.mappingPath[0] === 'catalogNumber');
+
   return treeRanksLoaded ? (
     <ReadOnlyContext.Provider value={isReadOnly}>
       <IsQueryBasicContext.Provider value={isBasic}>
@@ -556,6 +562,8 @@ function Wrapped({
               />
               <QueryToolbar
                 isDistinct={query.selectDistinct ?? false}
+                isSeries={query.selectSeries ?? false}
+                showSeries={showSeries}
                 showHiddenFields={showHiddenFields}
                 tableName={table.name}
                 onRunCountOnly={(): void => runQuery('count')}
@@ -570,6 +578,12 @@ function Wrapped({
                     selectDistinct: !(query.selectDistinct ?? false),
                   })
                 }
+                onToggleSeries={(): void =>
+                  setQuery({
+                    ...query,
+                    selectSeries: !(query.selectSeries ?? false),
+                  })
+                }
                 onToggleHidden={setShowHiddenFields}
               />
             </div>

specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx

@@ -97,6 +97,7 @@ export function createQuery(
   query.set('contextName', table.name);
   query.set('contextTableId', table.tableId);
   query.set('selectDistinct', false);
+  query.set('selectSeries', false);
   query.set('countOnly', false);
   query.set('formatAuditRecIds', false);
   query.set('specifyUser', userInformation.resource_uri);

specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts

@@ -35,6 +35,7 @@ export function makeComboBoxQuery({
   query.set('contextName', table.name);
   query.set('contextTableId', table.tableId);
   query.set('selectDistinct', false);
+  query.set('selectSeries', false);
   query.set('countOnly', false);
   query.set('specifyUser', userInformation.resource_uri);
   query.set('isFavorite', false);

specifyweb/frontend/js_src/lib/localization/query.ts

@@ -303,6 +303,9 @@ export const queryText = createDictionary({
     'uk-ua': 'Виразний',
     'de-ch': 'Unterscheidbar',
   },
+  series: {
+    'en-us': 'Series',
+  },
   createCsv: {
     'en-us': 'Create CSV',
     'ru-ru': 'Создать CSV-файл',

specifyweb/stored_queries/execution.py

@@ -26,6 +26,7 @@
 from ..permissions.permissions import check_table_permissions
 from ..specify.auditlog import auditlog
 from ..specify.models import Loan, Loanpreparation, Loanreturnpreparation, Taxontreedef
+from specifyweb.specify.utils import log_sqlalchemy_query
 
 logger = logging.getLogger(__name__)
 
@@ -130,7 +131,6 @@ def filter_by_collection(model, query, collection):
     return query
 
 
-
 EphemeralField = namedtuple('EphemeralField', "stringId isRelFld operStart startValue isNot isDisplay sortType formatName isStrict")
 
 def field_specs_from_json(json_fields):
@@ -405,6 +405,7 @@ def run_ephemeral_query(collection, user, spquery):
     offset = spquery.get('offset', 0)
     recordsetid = spquery.get('recordsetid', None)
     distinct = spquery['selectdistinct']
+    series = spquery.get('selectseries', None)
     tableid = spquery['contexttableid']
     count_only = spquery['countonly']
     try:
@@ -414,7 +415,7 @@ def run_ephemeral_query(collection, user, spquery):
 
     with models.session_context() as session:
         field_specs = field_specs_from_json(spquery['fields'])
-        return execute(session, collection, user, tableid, distinct, count_only,
+        return execute(session, collection, user, tableid, distinct, series, count_only,
                        field_specs, limit, offset, recordsetid, formatauditobjs=format_audits)
 
 def augment_field_specs(field_specs, formatauditobjs=False):
@@ -545,24 +546,35 @@ def return_loan_preps(collection, user, agent, data):
                 ])
         return to_return
 
-def execute(session, collection, user, tableid, distinct, count_only, field_specs, limit, offset, recordsetid=None, formatauditobjs=False):
+def execute(session, collection, user, tableid, distinct, series, count_only,
+            field_specs, limit, offset, recordsetid=None, formatauditobjs=False):
     "Build and execute a query, returning the results as a data structure for json serialization"
 
     set_group_concat_max_len(session.connection())
-    query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, formatauditobjs=formatauditobjs, distinct=distinct)
+    query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid,
+                                        formatauditobjs=formatauditobjs, distinct=distinct, series=series)
 
     if count_only:
         return {'count': query.count()}
     else:
         logger.debug("order by: %s", order_by_exprs)
+        if series: # maybe add - and catalog_number_field exists, and id_field doen't exist
+            query = query.order_by('catalognumber')
+
         query = query.order_by(*order_by_exprs).offset(offset)
+
         if limit:
             query = query.limit(limit)
 
+        if series:
+            return {'results': series_post_query(query)}
+
+        log_sqlalchemy_query(query) # Debugging
         return {'results': list(query)}
 
 def build_query(session, collection, user, tableid, field_specs,
-                recordsetid=None, replace_nulls=False, formatauditobjs=False, distinct=False, implicit_or=True):
+                recordsetid=None, replace_nulls=False, formatauditobjs=False,
+                distinct=False, series=False, implicit_or=True):
     """Build a sqlalchemy query using the QueryField objects given by
     field_specs.
 
@@ -587,18 +599,30 @@ def build_query(session, collection, user, tableid, field_specs,
     replace_nulls = if True, replace null values with ""
 
     distinct = if True, group by all display fields, and return all record IDs associated with a row
+
+    series = (only for Collection Objects) if True, group by all display fields.
+    Group consecutive catalog numbers together and return all record IDs associated with each grouped row.
     """
     model = models.models_by_tableid[tableid]
     id_field = getattr(model, model._id)
+    catalog_number_field = model.catalogNumber if hasattr(model, 'catalogNumber') else None
 
     field_specs = [apply_absolute_date(field_spec) for field_spec in field_specs]
     field_specs = [apply_specify_user_name(field_spec, user) for field_spec in field_specs]
 
-
+    query_construct_query = None
+    if series and catalog_number_field:
+        query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=','),
+                                              func.group_concat(catalog_number_field.distinct(), separator=','))
+    elif distinct:
+        query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=','))
+    else:
+        query_construct_query = query_construct_query = session.query(id_field)
+
     query = QueryConstruct(
         collection=collection,
         objectformatter=ObjectFormatter(collection, user, replace_nulls),
-        query=session.query(func.group_concat(id_field.distinct(), separator=',')) if distinct else session.query(id_field),
+        query=query_construct_query,
     )
 
     tables_to_read = set([
@@ -626,10 +650,13 @@ def build_query(session, collection, user, tableid, field_specs,
     order_by_exprs = []
     selected_fields = []
     predicates_by_field = defaultdict(list)
-    #augment_field_specs(field_specs, formatauditobjs)
+    # augment_field_specs(field_specs, formatauditobjs)
     for fs in field_specs:
         sort_type = SORT_TYPES[fs.sort_type]
 
+        if series and fs.fieldspec.get_field().name.lower() == 'catalognumber':
+            continue
+
         query, field, predicate = fs.add_to_query(query, formatauditobjs=formatauditobjs)
         if fs.display:
             formatted_field = query.objectformatter.fieldformat(fs, field)
@@ -656,11 +683,89 @@ def build_query(session, collection, user, tableid, field_specs,
         where = reduce(sql.and_, (p for ps in predicates_by_field.values() for p in ps))
         query = query.filter(where)
 
-    if distinct:
+    if series:
+        query = group_by_displayed_fields(query, selected_fields, ignore_cat_num=True)
+    elif distinct:
         query = group_by_displayed_fields(query, selected_fields)
 
     internal_predicate = query.get_internal_filters()
     query = query.filter(internal_predicate)
 
     logger.warning("query: %s", query.query)
     return query.query, order_by_exprs
+
+def series_post_query(query, co_id_col_index = 0, co_cat_num_col_index = 1):
+    def process_group_by_result(group_by_query_result, id_col_index = 0, group_col_index = 1):
+        def find_consecutive_ranges(lst):
+            def group_consecutives(acc, x):
+                if not acc or acc[-1][-1] + 1 != x:
+                    acc.append([x])
+                else:
+                    acc[-1].append(x)
+                return acc
+
+            grouped = reduce(group_consecutives, lst, [])
+
+            return [f"{g[0]:04d} - {g[-1]:04d}" if len(g) > 1 else f"{g[0]:04d}" for g in grouped]
+
+        def parse_numbers(num_str):
+            return sorted(map(int, filter(None, map(str.strip, num_str.replace(',', ' ').split()))))
+
+        def format_record(record):
+            id_part = record[id_col_index]
+            id_values = id_part.split(',')
+
+            num_ranges = find_consecutive_ranges(parse_numbers(record[group_col_index]))
+            formatted_records = [[id_values[0]] + [num_ranges[0]] + list(record[2:])] if len(id_values) == 1 else []
+
+            if len(num_ranges) > 1:
+                for num_range in num_ranges[1:]:
+                    formatted_records.append([id_values.pop()] + [num_range] + list(record[2:]))
+
+            return formatted_records if formatted_records else [[id_part] + [num_ranges[0]] + list(record[2:])]
+
+        formatted_records = [format_record(record[:]) for record in group_by_query_result]
+
+        result = [item for sublist in formatted_records for item in sublist]
+        result.sort(key=lambda x: int(x[1].split(' - ')[0]))
+
+        return result
+
+    return process_group_by_result(list(query), co_id_col_index, co_cat_num_col_index)
+
+def series_post_query_test(query): # REMOVE
+
+    query_results = list(query)
+    series_query_results = []
+
+    input = [["0012,0013,0014", "SomeText1", "Vial"],
+     ["0015", "OtherText", "Vial"],
+     ["0016,", "AnotherText", "Vial"],
+     ["0017,0018", "SomeText2", "Vial"],
+     ["0020, 0021, 0022", "SomeText3", "Vial"]]
+
+    output = [["0012 - 0014", "SomeText1", "Vial"],
+     ["0015", "OtherText", "Vial"],
+     ["0016", "AnotherText", "Vial"],
+     ["0017 - 0018", "SomeText2", "Vial"],
+     ["0020 - 0022", "SomeText3", "Vial"]]
+
+
+    input = [
+        ["1,2,3", "0021,0022,0043", "SomeText1", "Vial"],
+        ["4", "0023", "OtherText", "Vial"],
+        ["5", "0024", "AnotherText", "Vial"],
+        ["6,7", "0025,0026", "SomeText2", "Vial"],
+        ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"]
+    ]
+
+    output = [
+        ["1,2", "0021 - 0022", "SomeText1", "Vial"],
+        ["4", "0023", "OtherText", "Vial"],
+        ["5", "0024", "AnotherText", "Vial"],
+        ["6,7", "0025 - 0026", "SomeText2", "Vial"],
+        ["8,9,10", "0027 - 0029", "SomeText3", "Vial"],
+        ["3", "0043", "SomeText1", "Vial"]
+    ]
+
+    return series_query_results

specifyweb/stored_queries/group_concat.py

@@ -1,7 +1,6 @@
 # Based on stackoverflow answer from Wolph:
 # https://stackoverflow.com/questions/19205850/how-do-i-write-a-group-concat-function-in-sqlalchemy
 
-import re
 import sqlalchemy
 from sqlalchemy.sql import expression
 from sqlalchemy.ext import compiler
@@ -43,8 +42,16 @@ def process_clause(idx):
 
     return expr, separator, order_by
 
-def group_by_displayed_fields(query: QueryConstruct, fields):
+def group_by_displayed_fields(query: QueryConstruct, fields, ignore_cat_num=False):
     for field in fields:
+        if (
+            ignore_cat_num
+            and hasattr(field, "clause")
+            and field.clause is not None
+            and hasattr(field.clause, "key")
+            and field.clause.key == "CatalogNumber"
+        ):
+            continue
         query = query.group_by(field)
-    
+
     return query

specifyweb/stored_queries/views.py

@@ -82,14 +82,15 @@ def query(request, id):
     with models.session_context() as session:
         sp_query = session.query(models.SpQuery).get(int(id))
         distinct = sp_query.selectDistinct
+        series = sp_query.selectSeries if sp_query.selectSeries else None
         tableid = sp_query.contextTableId
         count_only = sp_query.countOnly
 
         field_specs = [QueryField.from_spqueryfield(field, value_from_request(field, request.GET))
                        for field in sorted(sp_query.fields, key=lambda field: field.position)]
 
         data = execute(session, request.specify_collection, request.specify_user,
-                       tableid, distinct, count_only, field_specs, limit, offset)
+                       tableid, distinct, series, count_only, field_specs, limit, offset)
 
     return HttpResponse(toJson(data), content_type='application/json')