Add series tool in query builder

specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts

@@ -123,6 +123,7 @@ describe('serializeResource', () => {
       remarks: null,
       searchSynonymy: null,
       selectDistinct: null,
+      selectSeries: null,
       smushed: null,
       specifyUser: null,
       sqlStr: null,

specifyweb/frontend/js_src/lib/components/DataModel/types.ts

@@ -5291,6 +5291,7 @@ export type SpQuery = {
     readonly remarks: string | null;
     readonly searchSynonymy: boolean | null;
     readonly selectDistinct: boolean | null;
+    readonly selectSeries: boolean | null;
     readonly smushed: boolean | null;
     readonly sqlStr: string | null;
     readonly timestampCreated: string;

specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx

@@ -105,6 +105,7 @@ overrideAjax(
       resource_uri: undefined,
       searchsynonymy: null,
       selectdistinct: false,
+      selectseries: false,
       smushed: null,
       specifyuser: '/api/specify/specifyuser/2/',
       sqlstr: null,

specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx

@@ -13,16 +13,22 @@ export function QueryToolbar({
   showHiddenFields,
   tableName,
   isDistinct,
+  isSeries,
+  showSeries,
   onToggleHidden: handleToggleHidden,
   onToggleDistinct: handleToggleDistinct,
+  onToggleSeries: handleToggleSeries,
   onRunCountOnly: handleRunCountOnly,
   onSubmitClick: handleSubmitClick,
 }: {
   readonly showHiddenFields: boolean;
   readonly tableName: keyof Tables;
   readonly isDistinct: boolean;
+  readonly isSeries: boolean;
+  readonly showSeries: boolean;
   readonly onToggleHidden: (value: boolean) => void;
   readonly onToggleDistinct: () => void;
+  readonly onToggleSeries: () => void;
   readonly onRunCountOnly: () => void;
   readonly onSubmitClick: () => void;
 }): JSX.Element {
@@ -38,6 +44,16 @@ export function QueryToolbar({
       <span className="-ml-2 flex-1" />
       {hasPermission('/querybuilder/query', 'execute') && (
         <>
+          {showSeries && (
+            <Label.Inline>
+              <Input.Checkbox
+                checked={isSeries}
+                isReadOnly={isDistinct}
+                onChange={handleToggleSeries}
+              />
+              {queryText.series()}
+            </Label.Inline>
+          )}
           {/*
            * Query Distinct for trees is disabled because of
            * https://github.com/specify/specify7/pull/1019#issuecomment-973525594
@@ -46,6 +62,7 @@ export function QueryToolbar({
             <Label.Inline>
               <Input.Checkbox
                 checked={isDistinct}
+                isReadOnly={isSeries}
                 onChange={handleToggleDistinct}
               />
               {queryText.distinct()}

specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx

@@ -94,6 +94,7 @@ function Wrapped({
   readonly onChange?: (props: {
     readonly fields: RA<SerializedResource<SpQueryField>>;
     readonly isDistinct: boolean | null;
+    readonly isSeries: boolean | null;
   }) => void;
 }): JSX.Element {
   const [query, setQuery] = useResource(queryResource);
@@ -157,8 +158,9 @@ function Wrapped({
     handleChange?.({
       fields: unParseQueryFields(state.baseTableName, state.fields),
       isDistinct: query.selectDistinct,
+      isSeries: query.selectSeries,
     });
-  }, [state, query.selectDistinct]);
+  }, [state, query.selectDistinct, query.selectSeries]);
 
   /**
    * If tried to save a query, enforce the field length limit for the
@@ -296,6 +298,10 @@ function Wrapped({
     undefined
   );
 
+  const showSeries =
+    table.name === 'CollectionObject' &&
+    state.fields.some((field) => field.mappingPath[0] === 'catalogNumber');
+
   return treeRanksLoaded ? (
     <ReadOnlyContext.Provider value={isReadOnly}>
       <IsQueryBasicContext.Provider value={isBasic}>
@@ -556,6 +562,8 @@ function Wrapped({
               />
               <QueryToolbar
                 isDistinct={query.selectDistinct ?? false}
+                isSeries={query.selectSeries ?? false}
+                showSeries={showSeries}
                 showHiddenFields={showHiddenFields}
                 tableName={table.name}
                 onRunCountOnly={(): void => runQuery('count')}
@@ -570,6 +578,12 @@ function Wrapped({
                     selectDistinct: !(query.selectDistinct ?? false),
                   })
                 }
+                onToggleSeries={(): void =>
+                  setQuery({
+                    ...query,
+                    selectSeries: !(query.selectSeries ?? false),
+                  })
+                }
                 onToggleHidden={setShowHiddenFields}
               />
             </div>

specifyweb/frontend/js_src/lib/components/QueryBuilder/__tests__/__snapshots__/fromTree.test.ts.snap

@@ -86,6 +86,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Los Angeles County\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
   {
@@ -160,6 +161,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Cabinet 1\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
   {
@@ -234,6 +236,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Carpiodes velifer\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
   {
@@ -308,6 +311,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Paleocene\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
   {
@@ -382,6 +386,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Cretaceous\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
   {
@@ -456,6 +461,7 @@ exports[`queryFromTree 1`] = `
     "name": "Collection Object using \\"Plate\\"",
     "ordinal": 32767,
     "selectdistinct": false,
+    "selectseries": false,
     "specifyuser": "/api/specify/specifyuser/2/",
   },
 ]

specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx

@@ -97,6 +97,7 @@ export function createQuery(
   query.set('contextName', table.name);
   query.set('contextTableId', table.tableId);
   query.set('selectDistinct', false);
+  query.set('selectSeries', false);
   query.set('countOnly', false);
   query.set('formatAuditRecIds', false);
   query.set('specifyUser', userInformation.resource_uri);

specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts

@@ -35,6 +35,7 @@ export function makeComboBoxQuery({
   query.set('contextName', table.name);
   query.set('contextTableId', table.tableId);
   query.set('selectDistinct', false);
+  query.set('selectSeries', false);
   query.set('countOnly', false);
   query.set('specifyUser', userInformation.resource_uri);
   query.set('isFavorite', false);

specifyweb/frontend/js_src/lib/localization/query.ts

@@ -303,6 +303,9 @@ export const queryText = createDictionary({
     'uk-ua': 'Виразний',
     'de-ch': 'Unterscheidbar',
   },
+  series: {
+    'en-us': 'Series',
+  },
   createCsv: {
     'en-us': 'Create CSV',
     'ru-ru': 'Создать CSV-файл',

specifyweb/stored_queries/execution.py

@@ -10,7 +10,7 @@
 
 from django.conf import settings
 from django.db import transaction
-from sqlalchemy import sql, orm, func, select
+from sqlalchemy import sql, orm, func, select, text
 from sqlalchemy.sql.expression import asc, desc, insert, literal
 
 from specifyweb.stored_queries.group_concat import group_by_displayed_fields
@@ -26,6 +26,7 @@
 from ..permissions.permissions import check_table_permissions
 from ..specify.auditlog import auditlog
 from ..specify.models import Loan, Loanpreparation, Loanreturnpreparation, Taxontreedef
+from specifyweb.specify.utils import log_sqlalchemy_query
 
 logger = logging.getLogger(__name__)
 
@@ -130,7 +131,6 @@ def filter_by_collection(model, query, collection):
     return query
 
 
-
 EphemeralField = namedtuple('EphemeralField', "stringId isRelFld operStart startValue isNot isDisplay sortType formatName isStrict")
 
 def field_specs_from_json(json_fields):
@@ -405,6 +405,7 @@ def run_ephemeral_query(collection, user, spquery):
     offset = spquery.get('offset', 0)
     recordsetid = spquery.get('recordsetid', None)
     distinct = spquery['selectdistinct']
+    series = spquery.get('selectseries', None)
     tableid = spquery['contexttableid']
     count_only = spquery['countonly']
     try:
@@ -414,7 +415,7 @@ def run_ephemeral_query(collection, user, spquery):
 
     with models.session_context() as session:
         field_specs = field_specs_from_json(spquery['fields'])
-        return execute(session, collection, user, tableid, distinct, count_only,
+        return execute(session, collection, user, tableid, distinct, series, count_only,
                        field_specs, limit, offset, recordsetid, formatauditobjs=format_audits)
 
 def augment_field_specs(field_specs, formatauditobjs=False):
@@ -545,24 +546,35 @@ def return_loan_preps(collection, user, agent, data):
                 ])
         return to_return
 
-def execute(session, collection, user, tableid, distinct, count_only, field_specs, limit, offset, recordsetid=None, formatauditobjs=False):
+def execute(session, collection, user, tableid, distinct, series, count_only,
+            field_specs, limit, offset, recordsetid=None, formatauditobjs=False):
     "Build and execute a query, returning the results as a data structure for json serialization"
 
     set_group_concat_max_len(session.connection())
-    query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, formatauditobjs=formatauditobjs, distinct=distinct)
+    query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid,
+                                        formatauditobjs=formatauditobjs, distinct=distinct, series=series)
 
     if count_only:
         return {'count': query.count()}
     else:
+        if series:
+            order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))"))
+
         logger.debug("order by: %s", order_by_exprs)
         query = query.order_by(*order_by_exprs).offset(offset)
+
         if limit:
             query = query.limit(limit)
 
+        if series:
+            return {'results': series_post_query(query)}
+
+        log_sqlalchemy_query(query) # Debugging
         return {'results': list(query)}
 
 def build_query(session, collection, user, tableid, field_specs,
-                recordsetid=None, replace_nulls=False, formatauditobjs=False, distinct=False, implicit_or=True):
+                recordsetid=None, replace_nulls=False, formatauditobjs=False,
+                distinct=False, series=False, implicit_or=True):
     """Build a sqlalchemy query using the QueryField objects given by
     field_specs.
 
@@ -587,18 +599,39 @@ def build_query(session, collection, user, tableid, field_specs,
     replace_nulls = if True, replace null values with ""
 
     distinct = if True, group by all display fields, and return all record IDs associated with a row
+
+    series = (only for CO) if True, group by all display fields.
+    Group catalog numbers that fall within the same range together.
+    Return all record IDs associated with a row.
     """
     model = models.models_by_tableid[tableid]
     id_field = getattr(model, model._id)
+    catalog_number_field = model.catalogNumber if hasattr(model, 'catalogNumber') else None
 
     field_specs = [apply_absolute_date(field_spec) for field_spec in field_specs]
     field_specs = [apply_specify_user_name(field_spec, user) for field_spec in field_specs]
 
-
+    query_construct_query = None
+    if series and catalog_number_field:
+        query_construct_query = session.query(
+            func.group_concat(
+                func.concat(
+                    id_field,
+                    ':',
+                    catalog_number_field
+                ),
+                separator='|'
+            ).label('co_id_catnum_paired_values')
+        )
+    elif distinct:
+        query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=','))
+    else:
+        query_construct_query = query_construct_query = session.query(id_field)
+
     query = QueryConstruct(
         collection=collection,
         objectformatter=ObjectFormatter(collection, user, replace_nulls),
-        query=session.query(func.group_concat(id_field.distinct(), separator=',')) if distinct else session.query(id_field),
+        query=query_construct_query,
     )
 
     tables_to_read = set([
@@ -626,15 +659,22 @@ def build_query(session, collection, user, tableid, field_specs,
     order_by_exprs = []
     selected_fields = []
     predicates_by_field = defaultdict(list)
-    #augment_field_specs(field_specs, formatauditobjs)
+    # augment_field_specs(field_specs, formatauditobjs)
     for fs in field_specs:
         sort_type = SORT_TYPES[fs.sort_type]
 
+        if series and fs.fieldspec.get_field().name.lower() == 'catalognumber':
+            continue
+
         query, field, predicate = fs.add_to_query(query, formatauditobjs=formatauditobjs)
         if fs.display:
             formatted_field = query.objectformatter.fieldformat(fs, field)
             query = query.add_columns(formatted_field)
             selected_fields.append(formatted_field)
+
+        if hasattr(field, 'key') and field.key and field.key.lower() == 'catalognumber':
+            catalog_number_field = formatted_field
+
 
         if sort_type is not None:
             order_by_exprs.append(sort_type(field))
@@ -656,11 +696,52 @@ def build_query(session, collection, user, tableid, field_specs,
         where = reduce(sql.and_, (p for ps in predicates_by_field.values() for p in ps))
         query = query.filter(where)
 
-    if distinct:
+    if series:
+        query = group_by_displayed_fields(query, selected_fields, ignore_cat_num=True)
+    elif distinct:
         query = group_by_displayed_fields(query, selected_fields)
 
     internal_predicate = query.get_internal_filters()
     query = query.filter(internal_predicate)
 
     logger.warning("query: %s", query.query)
     return query.query, order_by_exprs
+
+def series_post_query(query, co_id_cat_num_pair_col_index=0):
+    """Transform the query results by removing the co_id:catnum pair column
+    and adding a co_id colum and formatted catnum range column.
+    Sort the results by the first catnum in the range."""
+    log_sqlalchemy_query(query)  # Debugging
+
+    def group_consecutive_ranges(lst):
+        def group_consecutives(acc, x):
+            if not acc or int(acc[-1][-1][1]) + 1 != int(x[1]):
+                acc.append([x])
+            else:
+                acc[-1].append(x)
+            return acc
+
+        grouped = reduce(group_consecutives, lst, [])
+        return [
+            (','.join([x[0] for x in group]), f"{group[0][1]} - {group[-1][1]}" if len(group) > 1 else f"{group[0][1]}")
+            for group in grouped
+        ]
+
+    def process_row(row):
+        co_id_cat_num_consecutive_pairs = group_consecutive_ranges(
+            sorted(
+                (pair.split(':') for pair in row[co_id_cat_num_pair_col_index].split(',')),
+                key=lambda x: int(x[1])
+            )
+        )
+
+        return [
+            [co_id, cat_num_series] + list(
+                list(row[1:]) if co_id_cat_num_pair_col_index == 0
+                else list(row[:co_id_cat_num_pair_col_index]) + list(row[co_id_cat_num_pair_col_index + 1:])
+            )
+            for co_id, cat_num_series in co_id_cat_num_consecutive_pairs
+        ]
+
+    MAX_ROWS = 500
+    return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS]