From ae7eb102ad9ac86e65aed69030fd35550a12d367 Mon Sep 17 00:00:00 2001 From: Richard Cox Date: Thu, 3 Aug 2023 18:02:11 +0100 Subject: [PATCH 01/12] Pagination Alpha --- shell/components/ResourceList/index.vue | 37 +++- shell/components/ResourceTable.vue | 25 ++- shell/components/SortableTable/filtering.js | 10 +- shell/components/SortableTable/index.vue | 62 +++++-- shell/components/SortableTable/paging.js | 33 +++- shell/components/SortableTable/sorting.js | 23 ++- shell/components/nav/NamespaceFilter.vue | 2 +- shell/config/pagination-table-headers.js | 110 ++++++++++++ shell/config/product/explorer.js | 99 ++++++++--- shell/mixins/resource-fetch-api-pagination.js | 167 ++++++++++++++++++ shell/mixins/resource-fetch-namespaced.js | 2 +- shell/mixins/resource-fetch.js | 10 +- shell/plugins/dashboard-store/actions.js | 29 ++- .../dashboard-store/dashboard-store.types.ts | 27 +++ shell/plugins/dashboard-store/getters.js | 23 +++ shell/plugins/dashboard-store/mutations.js | 33 +++- .../__tests__/utils/mutation.test.helpers.ts | 17 +- shell/plugins/steve/getters.js | 13 +- shell/plugins/steve/mutations.js | 5 +- shell/plugins/steve/pagination-utils.ts | 86 +++++++++ .../projectAndNamespaceFiltering.utils.ts | 40 +++-- .../steve/subscribe-namespace-handler.ts | 86 +++++++++ shell/plugins/steve/subscribe.js | 62 +------ shell/store/type-map.js | 139 +++++++++------ shell/utils/array.ts | 21 ++- 25 files changed, 961 insertions(+), 200 deletions(-) create mode 100644 shell/config/pagination-table-headers.js create mode 100644 shell/mixins/resource-fetch-api-pagination.js create mode 100644 shell/plugins/dashboard-store/dashboard-store.types.ts create mode 100644 shell/plugins/steve/pagination-utils.ts rename shell/{utils => plugins/steve}/projectAndNamespaceFiltering.utils.ts (55%) create mode 100644 shell/plugins/steve/subscribe-namespace-handler.ts diff --git a/shell/components/ResourceList/index.vue b/shell/components/ResourceList/index.vue index 39c9b51fa44..a7869e2470e 100644 --- a/shell/components/ResourceList/index.vue +++ b/shell/components/ResourceList/index.vue @@ -8,7 +8,8 @@ import IconMessage from '@shell/components/IconMessage.vue'; import { ResourceListComponentName } from './resource-list.config'; import { PanelLocation, ExtensionPoint } from '@shell/core/types'; import ExtensionPanel from '@shell/components/ExtensionPanel'; -import { sameContents } from '@shell/utils/array'; +import { sameArrayObjects, sameContents } from '@shell/utils/array'; +import { isEqual } from '@shell/utils/object'; export default { name: ResourceListComponentName, @@ -76,7 +77,8 @@ export default { } // See comment for `namespaceFilterRequired` watcher, skip fetch if we don't have a valid NS - if (!this.namespaceFilterRequired) { + // TODO: RC TODO tie in to comment above + if (!this.namespaceFilterRequired && !this.canPaginate) { await this.$fetchType(resource); } } @@ -124,6 +126,10 @@ export default { return []; } + if (this.pagination) { + return this.paginationHeaders; + } + return this.$store.getters['type-map/headersFor'](this.schema); }, @@ -155,7 +161,25 @@ export default { if (neu && !this.hasFetch) { this.$fetchType(this.resource); } - } + }, + + pagination(neu, old) { + const { filter: neuFilter, sort: neuSort, ...newPrimitiveTypes } = neu; + const { filter: oldFilter, sort: oldSort, ...oldPrimitiveTypes } = old; + + if ( + isEqual(newPrimitiveTypes, oldPrimitiveTypes) && + isEqual(neuFilter, oldFilter) && + sameArrayObjects(neuSort, oldSort) + ) { + // TODO: RC TEST more + return; + } + + if (neu && !this.hasFetch) { + this.$fetchType(this.resource); + } + }, }, created() { @@ -225,11 +249,14 @@ export default { :adv-filter-prevent-filtering-labels="advFilterPreventFilteringLabels" :use-query-params-for-simple-filtering="useQueryParamsForSimpleFiltering" :force-update-live-and-delayed="forceUpdateLiveAndDelayed" + :external-pagination="!!pagination" + :external-pagination-result="paginationResult" + @pagination-changed="paginationChanged" /> - + diff --git a/shell/components/ResourceTable.vue b/shell/components/ResourceTable.vue index 147fa10b4b9..f8831eb85fb 100644 --- a/shell/components/ResourceTable.vue +++ b/shell/components/ResourceTable.vue @@ -166,6 +166,16 @@ export default { forceUpdateLiveAndDelayed: { type: Number, default: 0 + }, + + externalPagination: { + type: Boolean, + default: false + }, + + externalPaginationResult: { + type: Object, + default: null } }, @@ -186,6 +196,7 @@ export default { data() { const options = this.$store.getters[`type-map/optionsFor`](this.schema); const listGroups = options?.listGroups || []; + const listGroupsWillOverride = options?.listGroupsWillOverride; const listGroupMapped = listGroups.reduce((acc, grp) => { acc[grp.value] = grp; @@ -196,7 +207,7 @@ export default { const inStore = this.schema?.id ? this.$store.getters['currentStore'](this.schema.id) : undefined; return { - listGroups, listGroupMapped, inStore + listGroups, listGroupsWillOverride, listGroupMapped, inStore }; }, @@ -236,7 +247,7 @@ export default { if ( this.headers ) { headers = this.headers.slice(); } else { - headers = this.$store.getters['type-map/headersFor'](this.schema); + headers = this.$store.getters['type-map/headersFor'](this.schema); // TODO: RC not required atm, but needs to be integrated with pagination headers } // add custom table columns provided by the extensions ExtensionPoint.TABLE_COL hook @@ -296,6 +307,9 @@ export default { return headers; }, + /** + * Take rows and filter out entries given the namespace filter + */ filteredRows() { const isAll = this.$store.getters['isAllNamespaces']; @@ -303,6 +317,7 @@ export default { if ( !this.isNamespaced || // Resource type isn't namespaced this.ignoreFilter || // Component owner strictly states no filtering + this.externalPagination || (isAll && !this.currentProduct?.hideSystemResources) || // Need all (this.inStore ? this.$store.getters[`${ this.inStore }/haveNamespace`](this.schema.id)?.length : false)// Store reports type has namespace filter, so rows already contain the correctly filtered resources ) { @@ -384,6 +399,10 @@ export default { }, groupOptions() { + if (this.listGroupsWillOverride && this.listGroups?.length) { + return this.listGroups; + } + const standard = [ { tooltipKey: 'resourceTable.groupBy.none', @@ -508,6 +527,8 @@ export default { :sort-generation-fn="safeSortGenerationFn" :use-query-params-for-simple-filtering="useQueryParamsForSimpleFiltering" :force-update-live-and-delayed="forceUpdateLiveAndDelayed" + :external-pagination="externalPagination" + :external-pagination-result="externalPaginationResult" @clickedActionButton="handleActionButtonClick" @group-value-change="group = $event" v-on="$listeners" diff --git a/shell/components/SortableTable/filtering.js b/shell/components/SortableTable/filtering.js index 61c32851b71..19f45514534 100644 --- a/shell/components/SortableTable/filtering.js +++ b/shell/components/SortableTable/filtering.js @@ -33,6 +33,10 @@ export default { }), */ filteredRows() { + if (this.externalPagination) { + return; + } + // PROP hasAdvancedFiltering comes from Advanced Filtering mixin (careful changing data var there...) if (!this.hasAdvancedFiltering) { return this.handleFiltering(); @@ -162,7 +166,11 @@ export default { arrangedRows(q) { // The rows changed so the old filter result is no longer useful this.previousResult = null; - } + }, + + searchQuery() { + this.debouncedPaginationChanged(); + }, }, }; diff --git a/shell/components/SortableTable/index.vue b/shell/components/SortableTable/index.vue index e455c221b4d..4978c0cb815 100644 --- a/shell/components/SortableTable/index.vue +++ b/shell/components/SortableTable/index.vue @@ -49,10 +49,12 @@ export const COLUMN_BREAKPOINTS = { // Data Flow: // rows prop -// -> arrangedRows (sorting.js) -// -> filteredRows (filtering.js) -// -> pagedRows (paging.js) -// -> groupedRows (grouping.js) +// --> sorting.js arrangedRows +// --> filtering.js handleFiltering() +// --> filtering.js filteredRows +// --> paging.js pageRows +// --> grouping.js groupedRows +// --> index.vue displayedRows export default { name: 'SortableTable', @@ -313,6 +315,15 @@ export default { forceUpdateLiveAndDelayed: { type: Number, default: 0 + }, + + externalPagination: { + type: Boolean, + default: false + }, + externalPaginationResult: { + type: Object, + default: null } }, @@ -327,12 +338,14 @@ export default { } return { - currentPhase: ASYNC_BUTTON_STATES.WAITING, - expanded: {}, + currentPhase: ASYNC_BUTTON_STATES.WAITING, + expanded: {}, searchQuery, eventualSearchQuery, - actionOfInterest: null, - loadingDelay: false, + subMatches: null, + actionOfInterest: null, + loadingDelay: false, + debouncedPaginationChanged: null, }; }, @@ -346,6 +359,10 @@ export default { this._onScroll = this.onScroll.bind(this); $main?.addEventListener('scroll', this._onScroll); + + if (this.externalPagination) { + this.debouncedPaginationChanged(); + } }, beforeDestroy() { @@ -429,6 +446,7 @@ export default { created() { this.debouncedRefreshTableData = debounce(this.refreshTableData, 500); + this.debouncedPaginationChanged = debounce(this.paginationChanged, 50); }, computed: { @@ -898,6 +916,26 @@ export default { event, targetElement: this.$refs[`actionButton${ i }`][0], }); + }, + + paginationChanged() { + // eslint-disable-next-line no-console + console.warn('ss', 'methods', 'paginationChanged', { + page: this.page, + perPage: this.perPage, + filter: this.searchFields, + sort: this.sortFields, + }); + this.$emit('pagination-changed', { + page: this.page, + perPage: this.perPage, + filter: { + searchFields: this.searchFields, + searchQuery: this.searchQuery + }, + sort: this.sortFields, + descending: this.descending + }); } } }; @@ -1443,7 +1481,7 @@ export default { - + - + diff --git a/shell/components/SortableTable/paging.js b/shell/components/SortableTable/paging.js index ed775e2a85e..7abcddeb1b5 100644 --- a/shell/components/SortableTable/paging.js +++ b/shell/components/SortableTable/paging.js @@ -2,16 +2,25 @@ import { ROWS_PER_PAGE } from '@shell/store/prefs'; export default { computed: { + totalRows() { + console.warn('ss', 'mixins', 'paging', 'computed', 'totalrows', this.externalPaginationResult); // eslint-disable-line no-console + if (this.externalPagination) { + return this.externalPaginationResult?.count || 0; + } + + return this.filteredRows.length; + }, + indexFrom() { return Math.max(0, 1 + this.perPage * (this.page - 1)); }, indexTo() { - return Math.min(this.filteredRows.length, this.indexFrom + this.perPage - 1); + return Math.min(this.totalRows, this.indexFrom + this.perPage - 1); }, totalPages() { - return Math.ceil(this.filteredRows.length / this.perPage ); + return Math.ceil(this.totalRows / this.perPage ); }, showPaging() { @@ -22,7 +31,7 @@ export default { const opt = { ...(this.pagingParams || {}), - count: this.filteredRows.length, + count: this.totalRows, pages: this.totalPages, from: this.indexFrom, to: this.indexTo, @@ -32,7 +41,10 @@ export default { }, pagedRows() { - if ( this.paging ) { + console.warn('sortable', 'mixin', 'paging', 'pagedRows', this.externalPagination); // eslint-disable-line no-console + if (this.externalPagination) { + return this.rows; + } else if ( this.paging ) { return this.filteredRows.slice(this.indexFrom - 1, this.indexTo); } else { return this.filteredRows; @@ -51,12 +63,21 @@ export default { // Go to the last page if we end up "past" the last page because the table changed const from = this.indexFrom; - const last = this.filteredRows.length; + const last = this.totalRows; if ( this.totalPages > 0 && this.page > 1 && from > last ) { this.setPage(this.totalPages); } - } + }, + + page() { + this.debouncedPaginationChanged(); + }, + + perPage() { + this.debouncedPaginationChanged(); + }, + }, methods: { diff --git a/shell/components/SortableTable/sorting.js b/shell/components/SortableTable/sorting.js index dc1e068d0cd..bea50eb89ad 100644 --- a/shell/components/SortableTable/sorting.js +++ b/shell/components/SortableTable/sorting.js @@ -23,13 +23,22 @@ export default { const out = [...fromGroup, ...fromColumn]; - addObject(out, 'nameSort'); + if (this.externalPagination) { + addObject(out, 'metadata.name'); // TODO: RC FIX its a steve things + } else { + addObject(out, 'nameSort'); + } + addObject(out, 'id'); return out; }, arrangedRows() { + if (this.externalPagination) { + return; + } + let key; if ( this.sortGenerationFn ) { @@ -101,4 +110,16 @@ export default { this.setPage(1); }, }, + + watch: { + sortFields(neu) { + console.warn('paging', 'watch', 'sortFields', neu); // eslint-disable-line no-console + this.debouncedPaginationChanged(); + }, + + descending(neu) { + console.warn('paging', 'watch', 'descending', neu); // eslint-disable-line no-console + this.debouncedPaginationChanged(); + } + } }; diff --git a/shell/components/nav/NamespaceFilter.vue b/shell/components/nav/NamespaceFilter.vue index b13228b67b5..3090bee82b8 100644 --- a/shell/components/nav/NamespaceFilter.vue +++ b/shell/components/nav/NamespaceFilter.vue @@ -17,7 +17,7 @@ import { NAMESPACE_FILTER_P_FULL_PREFIX, } from '@shell/utils/namespace-filter'; import { KEY } from '@shell/utils/platform'; -import pAndNFiltering from '@shell/utils/projectAndNamespaceFiltering.utils'; +import pAndNFiltering from '@shell/plugins/steve/projectAndNamespaceFiltering.utils'; import { SETTING } from '@shell/config/settings'; const forcedNamespaceValidTypes = [NAMESPACE_FILTER_KINDS.DIVIDER, NAMESPACE_FILTER_KINDS.PROJECT, NAMESPACE_FILTER_KINDS.NAMESPACE]; diff --git a/shell/config/pagination-table-headers.js b/shell/config/pagination-table-headers.js new file mode 100644 index 00000000000..86403acafb4 --- /dev/null +++ b/shell/config/pagination-table-headers.js @@ -0,0 +1,110 @@ +import { STATE, NAME as NAME_COL, NAMESPACE as NAMESPACE_COL, AGE } from '@shell/config/table-headers'; + +// These contain paths set to specific values within a resource returned by the server +// In the future specific resource types will extend these with their own definitions +// For tidyness that should be done alongside their models + +export const STEVE_NAME_COL = { + ...NAME_COL, + value: 'metadata.name', + sort: ['metadata.name'], + search: 'metadata.name', +}; + +export const STEVE_STATE_COL = { + ...STATE, + // value: 'metadata.state.name', // This means what the user sees does not align with what column is sorted or filtered on + // formatter: null + sort: ['metadata.state.name'], + search: 'metadata.state.name', +}; + +export const STEVE_AGE_COL = { + ...AGE, + value: 'metadata.creationTimestamp', + sort: 'metadata.creationTimestamp:desc', + search: false, +}; + +export const STEVE_NAMESPACE_COL = { + ...NAMESPACE_COL, + value: 'metadata.namespace', + sort: 'metadata.namespace', + search: 'metadata.namespace', +}; + +export const STEVE_LIST_GROUPS = [{ + tooltipKey: 'resourceTable.groupBy.none', + icon: 'icon-list-flat', + value: 'none', +}, { + icon: 'icon-folder', + value: 'metadata.namespace', + field: 'metadata.namespace', // Default groupByLabel field in models is NS based + hideColumn: NAMESPACE_COL.name, + tooltipKey: 'resourceTable.groupBy.namespace' +}]; + +// TODO: RC TODO (can be later than alpha) +// - loading indicator when pages are being requested (take care not to blip) +// - performance setting (global enable / disable) +// - improve - the list group by feature adds a namespace option if resource is namespaced. we replace this with config for paths via `listGroupsWillOverride`. this isn't so neat +// - improve - there's a lot of computed properties in sortable table that fire when things happen in store. need to avoid these +// - we re-fetch the list when we return to the page. this means the list is updated, but user has to wait for http request again. could leave as is, candidate for refresh button +// - resolve two `TODO: RC FIX` + +// TODO: RC TODO (should be part of alpha) +// - add/update comments... everywhere +// - remove debug consoles +// - there are two subscribe messages sent for the list's type +// - there's no label on the namespace groups. this is due to the `field` used to search instead of the `value` + +// TODO: RC Comment +// - only applies to +// - cluster store. management store should in theory be simple to support +// - resources that don't have their own custom list. this should be easy to resolve though +// - specifically configmaps, secrets and pods. other resources can be incrementally added +// - sorting / filtering are disabled for... +// - any column who's value was computed locally (model getters). +// - state - we convert some `metadata.state.name` values to something more readable +// - secret - subTypeDisplay - translates things like `kubernetes.io/service-account-token` to `Svc Acct Token` +// - configmaps - data - we convert data from two properties into something more readable +// - values that are within an object in an array +// - pod - Container image - spec.container[0].image +// - columns that were originally from schemas cannot +// - pod - `Ready`, `Restarts` and `IP` all come from schema --> metadata.fields[index] (attributes.columns[0].field: "$.metadata.fields[1]) + +// - Design / Patterns +// - We can only sort and filter by values known to the backend. Therefore a lot of the existing table headers, which reference model properties +// are invalid. To get around this we define new headers which refer directly to properties on the raw resource. see pagination-table-headers +// this means we avoid things like `name` & `namespace` property helpers, searchFields that are functions to values, etc +// - the revision of the list is ignored. when user goes to page two it will be page 2 at that new time (rather than from the time page 1 was requested) +// - Question. the only way for a user to update their page is to change pagination in some way. we should consider a refresh button + +// TODO: RC Backend / API Issues +// - configmaps +// - different `count` provided when adding/removing `sort=-metadata.namespace` +// - configmaps?page=1&pagesize=10&sort=-metadata.namespace,-metadata.name,-id&filter=metadata.name=kube. count:14 +// - configmaps?page=2&pagesize=10&sort=-metadata.namespace,-metadata.name,-id&filter=metadata.name=kube. count:14 +// - configmaps?page=1&pagesize=10&sort=-metadata.name,-id&filter=metadata.name=kube,metadata.namespace=kube. count:18 +// - configmaps?page=2&pagesize=10&sort=-metadata.name,-id&filter=metadata.name=kube,metadata.namespace=kube. count:18 +// - secrets +// - the resource has a `_type` field, yet `sort=_type` does not work +// - the resource has a `_type` field, yet `filter_type` does not work +// - pod +// - question. spec.containers is an array of container objects. we'd like to sort/search using container.image +// - question. some columns come from schemas (pod 'ready' field is from schema..`attributes.columns[0].field: "$.metadata.fields[1]"` ). the field is a specific value in an array. can we sort / filter by it? + +// TODO: RC Test cases +// - no regressions for +// - non-paginated lists +// - namespace/project filtering +// - manual refresh (list) +// - incremental loading (list) +// - garbage collection (?) +// - advanced filtering +// - fleet workspace selection +// - group by namespace / no namespace +// - group by namespace sticks on refresh +// - the hideSystemResources product config (determines namespaces used in filters) +// - the pref ALL_NAMESPACES (determines namespaces used in filters) diff --git a/shell/config/product/explorer.js b/shell/config/product/explorer.js index ccdf1c1f2d5..13457c79a3e 100644 --- a/shell/config/product/explorer.js +++ b/shell/config/product/explorer.js @@ -19,10 +19,13 @@ import { STORAGE_CLASS_PROVISIONER, PERSISTENT_VOLUME_SOURCE, HPA_REFERENCE, MIN_REPLICA, MAX_REPLICA, CURRENT_REPLICA, ACCESS_KEY, DESCRIPTION, EXPIRES, EXPIRY_STATE, SUB_TYPE, AGE_NORMAN, SCOPE_NORMAN, PERSISTENT_VOLUME_CLAIM, RECLAIM_POLICY, PV_REASON, WORKLOAD_HEALTH_SCALE, POD_RESTARTS, - DURATION, MESSAGE, REASON, LAST_SEEN_TIME, EVENT_TYPE, OBJECT, ROLE, + DURATION, MESSAGE, REASON, LAST_SEEN_TIME, EVENT_TYPE, OBJECT, ROLE } from '@shell/config/table-headers'; import { DSL } from '@shell/store/type-map'; +import { + STEVE_AGE_COL, STEVE_LIST_GROUPS, STEVE_NAMESPACE_COL, STEVE_NAME_COL, STEVE_STATE_COL +} from '@shell/config/pagination-table-headers'; export const NAME = 'explorer'; @@ -169,19 +172,6 @@ export function init(store) { configureType(EVENT, { limit: 500 }); weightType(EVENT, -1, true); - // Allow Pods to be grouped by node - configureType(POD, { - listGroups: [ - { - icon: 'icon-cluster', - value: 'role', - field: 'groupByNode', - hideColumn: 'groupByNode', - tooltipKey: 'resourceTable.groupBy.node' - } - ] - }); - setGroupDefaultType('serviceDiscovery', SERVICE); configureType(WORKLOAD, { @@ -198,20 +188,55 @@ export function init(store) { configureType(MANAGEMENT.PSA, { localOnly: true }); headers(PV, [STATE, NAME_COL, RECLAIM_POLICY, PERSISTENT_VOLUME_CLAIM, PERSISTENT_VOLUME_SOURCE, PV_REASON, AGE]); - headers(CONFIG_MAP, [NAME_COL, NAMESPACE_COL, KEYS, AGE]); + + headers(CONFIG_MAP, + [NAME_COL, NAMESPACE_COL, KEYS, AGE], + [ + STEVE_NAME_COL, + STEVE_NAMESPACE_COL, { + ...KEYS, + sort: false, + search: false, + }, + STEVE_AGE_COL + ] + ); + configureType(CONFIG_MAP, { + listGroups: STEVE_LIST_GROUPS, + listGroupsWillOverride: true, + }); + + const secretData = { + name: 'data', + labelKey: 'tableHeaders.data', + value: 'dataPreview', + formatter: 'SecretData' + }; + headers(SECRET, [ STATE, NAME_COL, NAMESPACE_COL, SUB_TYPE, - { - name: 'data', - labelKey: 'tableHeaders.data', - value: 'dataPreview', - formatter: 'SecretData' - }, + secretData, AGE + ], [ + STEVE_STATE_COL, + STEVE_NAME_COL, + STEVE_NAMESPACE_COL, { + ...SUB_TYPE, + value: '_type', + sort: false, + search: false, + }, + secretData, + STEVE_AGE_COL ]); + configureType(SECRET, { + listGroups: STEVE_LIST_GROUPS, + listGroupsWillOverride: true, + }); + headers(INGRESS, [STATE, NAME_COL, NAMESPACE_COL, INGRESS_TARGET, INGRESS_DEFAULT_BACKEND, INGRESS_CLASS, AGE]); headers(SERVICE, [STATE, NAME_COL, NAMESPACE_COL, TARGET_PORT, SELECTOR, SPEC_TYPE, AGE]); headers(EVENT, [STATE, { ...LAST_SEEN_TIME, defaultSort: true }, EVENT_TYPE, REASON, OBJECT, 'Subobject', 'Source', MESSAGE, 'First Seen', 'Count', NAME_COL, NAMESPACE_COL]); @@ -224,7 +249,37 @@ export function init(store) { headers(WORKLOAD_TYPES.JOB, [STATE, NAME_COL, NAMESPACE_COL, WORKLOAD_IMAGES, WORKLOAD_ENDPOINTS, 'Completions', DURATION, POD_RESTARTS, AGE, WORKLOAD_HEALTH_SCALE]); headers(WORKLOAD_TYPES.CRON_JOB, [STATE, NAME_COL, NAMESPACE_COL, WORKLOAD_IMAGES, WORKLOAD_ENDPOINTS, 'Schedule', 'Last Schedule', POD_RESTARTS, AGE, WORKLOAD_HEALTH_SCALE]); headers(WORKLOAD_TYPES.REPLICATION_CONTROLLER, [STATE, NAME_COL, NAMESPACE_COL, WORKLOAD_IMAGES, WORKLOAD_ENDPOINTS, 'Ready', 'Current', 'Desired', POD_RESTARTS, AGE, WORKLOAD_HEALTH_SCALE]); - headers(POD, [STATE, NAME_COL, NAMESPACE_COL, POD_IMAGES, 'Ready', 'Restarts', 'IP', NODE_COL, AGE]); + + headers(POD, + [STATE, NAME_COL, NAMESPACE_COL, POD_IMAGES, 'Ready', 'Restarts', 'IP', NODE_COL, AGE], + [ + STEVE_STATE_COL, + STEVE_NAME_COL, + STEVE_NAMESPACE_COL, { + ...POD_IMAGES, + sort: false, + search: 'spec.containers.image' + }, 'Ready', 'Restarts', 'IP', { + ...NODE_COL, + search: 'spec.nodeName' + }, + STEVE_AGE_COL + ]); + configureType(POD, { + listGroups: [ + ...STEVE_LIST_GROUPS, + // Allow Pods to be grouped by node + { + icon: 'icon-cluster', + value: 'role', + field: 'spec.nodeName', + hideColumn: 'groupByNode', + tooltipKey: 'resourceTable.groupBy.node' + } + ], + listGroupsWillOverride: true, + }); + headers(MANAGEMENT.PSA, [STATE, NAME_COL, { ...DESCRIPTION, width: undefined diff --git a/shell/mixins/resource-fetch-api-pagination.js b/shell/mixins/resource-fetch-api-pagination.js new file mode 100644 index 00000000000..0297903d619 --- /dev/null +++ b/shell/mixins/resource-fetch-api-pagination.js @@ -0,0 +1,167 @@ +import { NAMESPACE_FILTER_ALL_SYSTEM, NAMESPACE_FILTER_ALL_USER } from '@shell/utils/namespace-filter'; +import { NAMESPACE } from '@shell/config/types'; +import { ALL_NAMESPACES } from '@shell/store/prefs'; +import { mapGetters } from 'vuex'; +import { ResourceListComponentName } from '../components/ResourceList/resource-list.config'; +import stevePaginationUtils from '@shell/plugins/steve/pagination-utils'; + +/** + * Companion mixin used with `resource-fetch` for `ResourceList` to determine if the user needs to filter the list by a single namespace + */ +export default { + + data() { + return { + forceUpdateLiveAndDelayed: 0, + // This of type `OptPagination` + pPagination: { + namespaces: undefined, + page: 1, + pageSize: 10, + sort: [], + filter: {} + } + }; + }, + + methods: { + haveAllPaginated(type) { + return this.$store.getters['haveAllPaginated'](type); + }, + + paginationChanged(event) { + this.pPagination = { + // namespaces: this.namespaceFilters, + ...this.pPagination, + page: event.page, + pageSize: event.perPage, + sort: event.sort?.map((field) => ({ + field, + asc: !event.descending + })), + filter: event.filter.searchQuery ? event.filter.searchFields.map((field) => ({ + field, + value: event.filter.searchQuery, + })) : [] + }; + console.warn('mixin', 'method', 'paginationChanged', this.pPagination); // eslint-disable-line no-console + } + }, + + computed: { + ...mapGetters(['currentProduct', 'namespaceFilters']), + + canPaginate() { + return this.__paginationRequired && !!this.paginationHeaders?.length; + }, + + /** + * Returns the namespace that requests should be filtered by + */ + pagination() { + console.warn('mixin', 'computed', 'pagination', this.canPaginate, this.pPagination); // eslint-disable-line no-console + + return this.canPaginate ? this.pPagination : ''; + }, + + paginationHeaders() { + return this.$store.getters['type-map/paginationHeadersFor'](this.schema); + }, + + /** + * Do we need to filter the list by a namespace? This will control whether the user is shown an error + * + * We shouldn't show an error on pages with resources that aren't namespaced + */ + __paginationRequired() { + if (!stevePaginationUtils.isEnabled({ rootGetters: this.$store.getters })) { + return false; + } + + return this.__areResourcesNamespaced; + }, + + paginationResult() { + return this.havePaginated?.result; + }, + + havePaginated() { + // Only currently works with cluster store, so no need to worry about the store the resources are in + return this.$store.getters[`cluster/havePaginated`](this.resource); + }, + + }, + + watch: { + namespaceFilters: { + immediate: true, + async handler(neu) { + const isAll = this.$store.getters['isAllNamespaces']; + + const namespaced = this.schema.attributes.namespaced; + const paginationRequires = this.canPaginate; + + if (!namespaced || !paginationRequires) { + return; + } + + const pref = this.$store.getters['prefs/get'](ALL_NAMESPACES); + const hideSystemResources = this.currentProduct.hideSystemResources; + + const allButHidingSystemResources = isAll && (hideSystemResources || pref); + + let namespaces = neu; + + const allNamespaces = this.$store.getters[`${ this.inStore }/all`](NAMESPACE); + + if (allButHidingSystemResources) { + // Determine the disallowed namespaces (rather than possibly thousands of allowed) + namespaces = allNamespaces + .filter((ns) => { + const isObscure = pref ? false : ns.isObscure; // Filter out Rancher system namespaces + const isSystem = hideSystemResources ? ns.isSystem : false; // Filter out Fleet system namespaces + + return isObscure || isSystem; + }) + .map((ns) => `-${ ns.name }`); + } else if (neu.length === 1) { + const allSystem = allNamespaces.filter((ns) => ns.isSystem); + + if (neu[0] === NAMESPACE_FILTER_ALL_SYSTEM) { + // get a list of all system namespaces + namespaces = allSystem.map((ns) => `${ ns.name }`); + } else if (neu[0] === NAMESPACE_FILTER_ALL_USER) { + // Determine the disallowed namespaces (rather than possibly thousands of allowed) + namespaces = allSystem.map((ns) => `-${ ns.name }`); + } + } + + this.pPagination = { + ...this.pPagination, + namespaces, + }; + } + }, + + async pagination(neu) { + console.warn('mixin', 'watch', 'pagination', this.pagination); // eslint-disable-line no-console + + // TODO: RC TEST - this shouldn't fire on lists that don't support pagination + + if (neu) { + // When a NS filter is required and the user selects a different one, kick off a new set of API requests + // + // ResourceList has two modes + // 1) ResourceList component handles API request to fetch resources + // 2) Custom list component handles API request to fetch resources + // + // This covers case 2 + if (this.$options.name !== ResourceListComponentName && !!this.$fetch) { + await this.$fetch(); + } + // Ensure any live/delayed columns get updated + this.forceUpdateLiveAndDelayed = new Date().getTime(); + } + } + }, +}; diff --git a/shell/mixins/resource-fetch-namespaced.js b/shell/mixins/resource-fetch-namespaced.js index e0af5c1fa1c..81ca53b24e6 100644 --- a/shell/mixins/resource-fetch-namespaced.js +++ b/shell/mixins/resource-fetch-namespaced.js @@ -1,7 +1,7 @@ import { NAMESPACE_FILTER_NS_PREFIX, NAMESPACE_FILTER_P_PREFIX } from '@shell/utils/namespace-filter'; import { mapGetters } from 'vuex'; import { ResourceListComponentName } from '../components/ResourceList/resource-list.config'; -import pAndNFiltering from '@shell/utils/projectAndNamespaceFiltering.utils'; +import pAndNFiltering from '@shell/plugins/steve/projectAndNamespaceFiltering.utils'; /** * Companion mixin used with `resource-fetch` for `ResourceList` to determine if the user needs to filter the list by a single namespace diff --git a/shell/mixins/resource-fetch.js b/shell/mixins/resource-fetch.js index 4821a9561ce..ad726ba09af 100644 --- a/shell/mixins/resource-fetch.js +++ b/shell/mixins/resource-fetch.js @@ -2,13 +2,17 @@ import { mapGetters } from 'vuex'; import { COUNT, MANAGEMENT } from '@shell/config/types'; import { SETTING, DEFAULT_PERF_SETTING } from '@shell/config/settings'; import ResourceFetchNamespaced from '@shell/mixins/resource-fetch-namespaced'; +import ResourceFetchApiPagination from '@shell/mixins/resource-fetch-api-pagination'; // Number of pages to fetch when loading incrementally const PAGES = 4; export default { - mixins: [ResourceFetchNamespaced], + mixins: [ + ResourceFetchNamespaced, + ResourceFetchApiPagination + ], data() { // fetching the settings related to manual refresh from global settings @@ -126,7 +130,9 @@ export default { const schema = this.$store.getters[`${ currStore }/schemaFor`](type); - if (schema?.attributes?.namespaced) { // Is this specific resource namespaced (could be primary or secondary resource)? + if (this.pagination) { + opt.pagination = this.pagination; + } else if (schema?.attributes?.namespaced) { // Is this specific resource namespaced (could be primary or secondary resource)? opt.namespaced = this.namespaceFilter; // namespaceFilter will only be populated if applicable for primary resource } diff --git a/shell/plugins/dashboard-store/actions.js b/shell/plugins/dashboard-store/actions.js index 21fee94cf24..d2f7aa31ea4 100644 --- a/shell/plugins/dashboard-store/actions.js +++ b/shell/plugins/dashboard-store/actions.js @@ -155,7 +155,20 @@ export default { } // No need to request the resources if we have them already - if ( opt.force !== true && (getters['haveAll'](type) || getters['haveAllNamespace'](type, opt.namespaced))) { + if ( + opt.force !== true && + (opt.pagination ? getters['haveAllPaginated'](type, opt.pagination) : true) && + (getters['haveAll'](type) || getters['haveAllNamespace'](type, opt.namespaced)) + ) { + // TODO: RC TEST that when returning to a list we don't re-fetch + const args = { + type, + revision: '', + // watchNamespace - used sometimes when we haven't fetched the results of a single namespace + // namespaced - used when we have fetched the result of a single namespace (see https://github.com/rancher/dashboard/pull/7329/files) + namespace: opt.watchNamespace || opt.namespaced + }; + if (opt.watch !== false ) { const args = { type, @@ -304,10 +317,17 @@ export default { commit('loadAll', { ctx, type, - data: out.data, - revision: out.revision, + data: out.data, + revision: out.revision, skipHaveAll, - namespace: opt.namespaced + namespace: opt.namespaced, + pagination: opt.pagination ? { + request: opt.pagination, + result: { + count: out.count, + pages: out.pages + } + } : undefined, }); } @@ -323,6 +343,7 @@ export default { type, revision: out.revision, namespace: opt.watchNamespace || opt.namespaced, // it could be either apparently + // TODO: RC watch // ToDo: SM namespaced is sometimes a boolean and sometimes a string, I don't see it as especially broken but we should refactor that in the future force: opt.forceWatch === true, }; diff --git a/shell/plugins/dashboard-store/dashboard-store.types.ts b/shell/plugins/dashboard-store/dashboard-store.types.ts new file mode 100644 index 00000000000..bd58e42164b --- /dev/null +++ b/shell/plugins/dashboard-store/dashboard-store.types.ts @@ -0,0 +1,27 @@ +/** + * Pagination settings sent to actions and persisted to store + */ +export interface OptPagination { + namespaces?: string[]; + page: number, + pageSize: number, + sort: { field: string, asc: boolean }[], + filter: { field: string, value: string }[] +} + +/** + * Object persisted to store + */ +export interface StorePagination { + request: OptPagination, + result: { + count: number, + pages: number + } +} + +export type FindAllOpt = { + [key: string]: any, + namespaced?: string[], + pagination?: OptPagination, +} diff --git a/shell/plugins/dashboard-store/getters.js b/shell/plugins/dashboard-store/getters.js index 4adc16cebd1..d6d62eabef2 100644 --- a/shell/plugins/dashboard-store/getters.js +++ b/shell/plugins/dashboard-store/getters.js @@ -294,12 +294,35 @@ export default { return false; }, + haveAllPaginated: (state, getters) => (type, pagination) => { + if (!pagination) { + return false; + } + + type = getters.normalizeType(type); + const entry = state.types[type]; + + if ( entry ) { + // TODO: RC FIXME confirm that pagination === entry.havePagination + + return !!entry.havePagination; + } + + return false; + }, + haveNamespace: (state, getters) => (type) => { type = getters.normalizeType(type); return state.types[type]?.haveNamespace || null; }, + havePaginated: (state, getters) => (type) => { + type = getters.normalizeType(type); + + return state.types[type]?.havePagination || null; + }, + haveSelector: (state, getters) => (type, selector) => { type = getters.normalizeType(type); const entry = state.types[type]; diff --git a/shell/plugins/dashboard-store/mutations.js b/shell/plugins/dashboard-store/mutations.js index f2114ba136e..bd6203cbee1 100644 --- a/shell/plugins/dashboard-store/mutations.js +++ b/shell/plugins/dashboard-store/mutations.js @@ -11,13 +11,14 @@ function registerType(state, type) { if ( !cache ) { cache = { - list: [], - haveAll: false, - haveSelector: {}, - haveNamespace: undefined, // If the cached list only contains resources for a namespace, this will contain the ns name - revision: 0, // The highest known resourceVersion from the server for this type - generation: 0, // Updated every time something is loaded for this type - loadCounter: 0, // Used to cancel incremental loads if the page changes during load + list: [], + haveAll: false, + haveSelector: {}, + haveNamespace: undefined, // If the cached list only contains resources for a namespace, this will contain the ns name + havePagination: undefined, + revision: 0, // The highest known resourceVersion from the server for this type + generation: 0, // Updated every time something is loaded for this type + loadCounter: 0, // Used to cancel incremental loads if the page changes during load }; // Not enumerable so they don't get sent back to the client for SSR @@ -117,6 +118,7 @@ export function forgetType(state, type) { cache.haveAll = false; cache.haveSelector = {}; cache.haveNamespace = undefined; + cache.havePagination = undefined; cache.revision = 0; cache.generation = 0; clear(cache.list); @@ -258,6 +260,7 @@ export function loadAll(state, { ctx, skipHaveAll, namespace, + pagination, revision }) { const { getters } = ctx; @@ -291,8 +294,20 @@ export function loadAll(state, { // Allow requester to skip setting that everything has loaded if (!skipHaveAll) { - cache.haveNamespace = namespace; - cache.haveAll = !namespace; + if (pagination) { + // havePagination is of type `StorePagination` + cache.havePagination = pagination; + cache.haveNamespace = undefined; + cache.haveAll = undefined; + } else if (namespace) { + cache.havePagination = false; + cache.haveNamespace = namespace; + cache.haveAll = false; + } else { + cache.havePagination = false; + cache.haveNamespace = false; + cache.haveAll = true; + } } return proxies; diff --git a/shell/plugins/steve/__tests__/utils/mutation.test.helpers.ts b/shell/plugins/steve/__tests__/utils/mutation.test.helpers.ts index 3446b8db9b2..3426175c01d 100644 --- a/shell/plugins/steve/__tests__/utils/mutation.test.helpers.ts +++ b/shell/plugins/steve/__tests__/utils/mutation.test.helpers.ts @@ -25,14 +25,15 @@ const createPod = () => create(POD); const createPodResource = (props = {}) => createResource(POD, props); const createCache = (props: any) => ({ - generation: 0, - haveAll: false, - haveNamespace: undefined, - haveSelector: {}, - list: [], - loadCounter: 0, - revision: 0, - map: new Map(), + generation: 0, + haveAll: false, + haveNamespace: undefined, + havePagination: undefined, + haveSelector: {}, + list: [], + loadCounter: 0, + revision: 0, + map: new Map(), ...props }); diff --git a/shell/plugins/steve/getters.js b/shell/plugins/steve/getters.js index 2f6787b4486..585432721a3 100644 --- a/shell/plugins/steve/getters.js +++ b/shell/plugins/steve/getters.js @@ -8,7 +8,8 @@ import HybridModel, { cleanHybridResources } from './hybrid-class'; import NormanModel from './norman-class'; import { urlFor } from '@shell/plugins/dashboard-store/getters'; import { normalizeType } from '@shell/plugins/dashboard-store/normalize'; -import pAndNFiltering from '@shell/utils/projectAndNamespaceFiltering.utils'; +import pAndNFiltering from '@shell/plugins/steve/projectAndNamespaceFiltering.utils'; +import stevePaginationUtils from '@shell/plugins/steve/pagination-utils'; import { parse } from '@shell/utils/url'; import { splitObjectPath } from '@shell/utils/string'; import { parseType } from '@shell/models/schema'; @@ -35,6 +36,16 @@ export default { const parsedUrl = parse(url); const isSteve = steveRegEx.test(parsedUrl.path); + // TODO: RC steve (mgmt steve) vs steve (proxy to kube) + + // Pagination + const stevePagination = stevePaginationUtils.checkAndCreateParam(opt); + + if (stevePagination) { + url += `${ (url.includes('?') ? '&' : '?') + stevePagination }`; + } + // End: Pagination + // labelSelector if ( opt.labelSelector ) { url += `${ url.includes('?') ? '&' : '?' }labelSelector=${ opt.labelSelector }`; diff --git a/shell/plugins/steve/mutations.js b/shell/plugins/steve/mutations.js index 8db5fe10a94..cabe44dc52f 100644 --- a/shell/plugins/steve/mutations.js +++ b/shell/plugins/steve/mutations.js @@ -123,7 +123,8 @@ export default { ctx, skipHaveAll, namespace, - revision + revision, + pagination }) { // Performance testing in dev and when env var is set if (process.env.dev && !!process.env.perfTest) { @@ -131,7 +132,7 @@ export default { } const proxies = loadAll(state, { - type, data, ctx, skipHaveAll, namespace, revision + type, data, ctx, skipHaveAll, namespace, revision, pagination }); // If we loaded a set of pods, then update the podsByNamespace cache diff --git a/shell/plugins/steve/pagination-utils.ts b/shell/plugins/steve/pagination-utils.ts new file mode 100644 index 00000000000..c20204b2bd2 --- /dev/null +++ b/shell/plugins/steve/pagination-utils.ts @@ -0,0 +1,86 @@ +import projectAndNamespaceFilteringUtils from '@shell/plugins/steve/projectAndNamespaceFiltering.utils'; +import { FindAllOpt } from '@shell/plugins/dashboard-store/dashboard-store.types'; +// import { getPerformanceSetting } from 'utils/settings'; + +/** + * Help functions for steve pagination + */ +class StevePaginationUtils { + /** + * Is pagination enabled at a global level + */ + isEnabled({ rootGetters }: any) { + const currentProduct = rootGetters['currentProduct']; + + // Only enable for the cluster store at the moment. In theory this should work in management as well, as they're both 'steve' stores + if (currentProduct?.inStore !== 'cluster') { + return false; + } + + // ... no perf setting yet + // const perfConfig = getPerformanceSetting(rootGetters); + + return true; + } + + checkAndCreateParam(opt: FindAllOpt): string | undefined { + if (!opt.pagination) { + return; + } + + console.warn('steve page utils', 'checkAndCreateParam', opt.pagination); // eslint-disable-line no-console + + const params: string[] = []; + + const namespaceParam = this.createNamespacesParam(opt); + + if (namespaceParam) { + params.push(namespaceParam); + } + + if (opt.pagination.page) { + params.push(`page=${ opt.pagination.page }`); + } else { + throw new Error(`A pagination request is required but no 'page' property provided: ${ JSON.stringify(opt) }`); + } + + if (opt.pagination.pageSize) { + params.push(`pagesize=${ opt.pagination.pageSize }`); + } else { + throw new Error(`A pagination request is required but no 'page' property provided: ${ JSON.stringify(opt) }`); + } + + if (opt.pagination.sort?.length) { + const joined = opt.pagination.sort + .map((s) => `${ s.asc ? '' : '-' }${ s.field }`) + .join(','); + + params.push(`sort=${ joined }`); + } + + if (opt.pagination.filter?.length) { + const joined = opt.pagination.filter + .map(({ field, value }) => `${ field }=${ value }`) + .join(','); + + params.push(`filter=${ joined }`); + } + + // Note - There is a `limit` property that is by default 100,000. This can be disabled by using `limit=-1`, + // but we shouldn't be fetching any pages big enough to exceed the default + + console.warn('steve page utils', 'checkAndCreateParam', 'res', params); // eslint-disable-line no-console + + return params.join('&'); + } + + private createNamespacesParam(opt: FindAllOpt): string | undefined { + if (!opt.pagination?.namespaces) { + return ''; + } + + return projectAndNamespaceFilteringUtils.createParam(opt.pagination?.namespaces); + } +} + +export default new StevePaginationUtils(); diff --git a/shell/utils/projectAndNamespaceFiltering.utils.ts b/shell/plugins/steve/projectAndNamespaceFiltering.utils.ts similarity index 55% rename from shell/utils/projectAndNamespaceFiltering.utils.ts rename to shell/plugins/steve/projectAndNamespaceFiltering.utils.ts index 4b37b344d87..6c361526712 100644 --- a/shell/utils/projectAndNamespaceFiltering.utils.ts +++ b/shell/plugins/steve/projectAndNamespaceFiltering.utils.ts @@ -1,7 +1,6 @@ import { NAMESPACE_FILTER_NS_FULL_PREFIX, NAMESPACE_FILTER_P_FULL_PREFIX } from '@shell/utils/namespace-filter'; import { getPerformanceSetting } from '@shell/utils/settings'; - -type Opt = { [key: string]: any, namespaced?: string[]} +import { FindAllOpt } from '@shell/plugins/dashboard-store/dashboard-store.types'; class ProjectAndNamespaceFiltering { static param = 'projectsornamespaces' @@ -9,7 +8,7 @@ class ProjectAndNamespaceFiltering { /** * Does the request `opt` definition require resources are fetched from a specific set namespaces/projects? */ - isApplicable(opt: Opt): boolean { + isApplicable(opt: FindAllOpt): boolean { return Array.isArray(opt.namespaced); } @@ -37,7 +36,7 @@ class ProjectAndNamespaceFiltering { /** * Check if `opt` requires resources from specific ns/projects, if so return the required query param (x=y) */ - checkAndCreateParam(opt: Opt): string { + checkAndCreateParam(opt: FindAllOpt): string { if (!this.isApplicable(opt)) { return ''; } @@ -45,17 +44,38 @@ class ProjectAndNamespaceFiltering { return this.createParam(opt.namespaced); } - private createParam(namespaceFilter: string[] | undefined): string { + public createParam(namespaceFilter: string[] | undefined): string { if (!namespaceFilter || !namespaceFilter.length) { return ''; } - const projectsOrNamespaces = namespaceFilter - .map((f) => f.replace(NAMESPACE_FILTER_NS_FULL_PREFIX, '') - .replace(NAMESPACE_FILTER_P_FULL_PREFIX, '')) - .join(','); + const namespaces = namespaceFilter.reduce((res, n) => { + const name = n + .replace(NAMESPACE_FILTER_NS_FULL_PREFIX, '') + .replace(NAMESPACE_FILTER_P_FULL_PREFIX, ''); + + if (name.startsWith('-')) { + res.exclude.push(n.substring(1, n.length)); + } else { + res.include.push(name); + } + + return res; + }, { include: [] as string[], exclude: [] as string[] }); + + let res = ''; + + console.warn('pAndNUtil', 'createParam', namespaces.include, namespaces.exclude); // eslint-disable-line no-console + + if (namespaces.include.length) { + res = `${ ProjectAndNamespaceFiltering.param }=${ namespaces.include.join(',') }`; + } + + if (namespaces.exclude.length) { + res = `${ ProjectAndNamespaceFiltering.param }!=${ namespaces.exclude.join(',') }`; + } - return `${ ProjectAndNamespaceFiltering.param }=${ projectsOrNamespaces }`; + return res; } } diff --git a/shell/plugins/steve/subscribe-namespace-handler.ts b/shell/plugins/steve/subscribe-namespace-handler.ts new file mode 100644 index 00000000000..e4f5558dc1d --- /dev/null +++ b/shell/plugins/steve/subscribe-namespace-handler.ts @@ -0,0 +1,86 @@ +import pAndNFiltering from '@shell/plugins/steve/projectAndNamespaceFiltering.utils'; + +/** + * Sockets will not be able to subscribe to more than one namespace. If this is requested we pretend to handle it + * - Changes to all resources are monitored (no namespace provided in sub) + * - We ignore any events not from a required namespace (we have the conversion of project --> namespaces already) + */ +class SubscribeNamespaceHandler { + typeIsNamespaced({ getters }: any, type: string): boolean { + return getters.haveNamespace(type)?.length > 0; + } + + typeIsPaginated({ getters }: any, type: string): boolean { + return !!getters.havePaginated(type); + } + + filteredNamespaces({ rootGetters }: any) { + // Note - activeNamespaceCache should be accurate for both namespace/project filtering and pagination namespace/project filtering + return rootGetters.activeNamespaceCache; + } + + /** + * Note - namespace can be a list of projects or namespaces + */ + subscribeNamespace(namespace: string[]) { + if (pAndNFiltering.isApplicable({ namespaced: namespace }) && namespace.length) { + return undefined; // AKA sub to everything + } + + return namespace; + } + + validChange({ getters, rootGetters }: any, type: string, data: any) { + if (this.typeIsNamespaced({ getters }, type)) { + const namespaces = this.filteredNamespaces({ rootGetters }); + + if (!namespaces[data.metadata.namespace]) { + return false; + } + } + + if (this.typeIsPaginated({ getters }, type)) { + const page = getters['all'](type); + + return !!page.find((pR: any) => pR.id === data.id); + } + + return true; + } + + validateBatchChange({ getters, rootGetters }: any, batch: { [key: string]: any}) { + const namespaces = this.filteredNamespaces({ rootGetters }); + + Object.entries(batch).forEach(([type, entries]) => { + if (this.typeIsNamespaced({ getters }, type)) { + const schema = getters.schemaFor(type); + + if (!schema?.attributes?.namespaced) { + return; + } + + Object.keys(entries).forEach((id) => { + const namespace = id.split('/')[0]; + + if (!namespace || !namespaces[namespace]) { + delete entries[id]; + } + }); + } + + if (this.typeIsPaginated({ getters }, type)) { + const page = getters['all'](type); + + Object.keys(entries).forEach((id) => { + if (!page.find((pR: any) => pR.id === id)) { + delete entries[id]; + } + }); + } + }); + + return batch; + } +} + +export default new SubscribeNamespaceHandler(); diff --git a/shell/plugins/steve/subscribe.js b/shell/plugins/steve/subscribe.js index e41a804f3cf..047783798f0 100644 --- a/shell/plugins/steve/subscribe.js +++ b/shell/plugins/steve/subscribe.js @@ -32,7 +32,7 @@ import { escapeHtml } from '@shell/utils/string'; import { keyForSubscribe } from '@shell/plugins/steve/resourceWatcher'; import { waitFor } from '@shell/utils/async'; import { WORKER_MODES } from './worker'; -import pAndNFiltering from '@shell/utils/projectAndNamespaceFiltering.utils'; +import namespaceHandler from './subscribe-namespace-handler'; import { BLANK_CLUSTER, STORE } from '@shell/store/store-types.js'; // minimum length of time a disconnect notification is shown @@ -205,66 +205,6 @@ export function equivalentWatch(a, b) { return true; } -/** - * Sockets will not be able to subscribe to more than one namespace. If this is requested we pretend to handle it - * - Changes to all resources are monitored (no namespace provided in sub) - * - We ignore any events not from a required namespace (we have the conversion of project --> namespaces already) - */ -const namespaceHandler = { - /** - * Note - namespace can be a list of projects or namespaces - */ - subscribeNamespace: (namespace) => { - if (pAndNFiltering.isApplicable({ namespaced: namespace }) && namespace.length) { - return undefined; // AKA sub to everything - } - - return namespace; - }, - - validChange: ({ getters, rootGetters }, type, data) => { - const haveNamespace = getters.haveNamespace(type); - - if (haveNamespace?.length) { - const namespaces = rootGetters.activeNamespaceCache; - - if (!namespaces[data.metadata.namespace]) { - return false; - } - } - - return true; - }, - - validateBatchChange: ({ getters, rootGetters }, batch) => { - const namespaces = rootGetters.activeNamespaceCache; - - Object.entries(batch).forEach(([type, entries]) => { - const haveNamespace = getters.haveNamespace(type); - - if (!haveNamespace?.length) { - return; - } - - const schema = getters.schemaFor(type); - - if (!schema?.attributes?.namespaced) { - return; - } - - Object.keys(entries).forEach((id) => { - const namespace = id.split('/')[0]; - - if (!namespace || !namespaces[namespace]) { - delete entries[id]; - } - }); - }); - - return batch; - } -}; - function queueChange({ getters, state, rootGetters }, { data, revision }, load, label) { const type = getters.normalizeType(data.type); diff --git a/shell/store/type-map.js b/shell/store/type-map.js index 1e9bccff8d5..ee51b6a5e89 100644 --- a/shell/store/type-map.js +++ b/shell/store/type-map.js @@ -264,7 +264,7 @@ export function DSL(store, product, module = 'type-map') { store.commit(`${ module }/groupBy`, { type, field }); }, - headers(type, headers) { + headers(type, headers, paginationHeaders = []) { headers.forEach((header) => { // If on the client, then use the value getter if there is one if (header.getValue) { @@ -277,6 +277,7 @@ export function DSL(store, product, module = 'type-map') { }); store.commit(`${ module }/headers`, { type, headers }); + store.commit(`${ module }/paginationHeaders`, { type, paginationHeaders }); }, hideBulkActions(type, field) { @@ -406,6 +407,7 @@ export const state = function() { typeOptions: [], groupBy: {}, headers: {}, + paginationHeaders: {}, hideBulkActions: {}, schemaGeneration: 1, cache: { @@ -503,17 +505,18 @@ export const getters = { optionsFor(state) { const def = { - isCreatable: true, - isEditable: true, - isRemovable: true, - showState: true, - showAge: true, - canYaml: true, - namespaced: null, - listGroups: [], - depaginate: false, - customRoute: undefined, - resourceEditMasthead: true, + isCreatable: true, + isEditable: true, + isRemovable: true, + showState: true, + showAge: true, + canYaml: true, + namespaced: null, + listGroups: [], + listGroupsWillOverride: false, + depaginate: false, + customRoute: undefined, + resourceEditMasthead: true, }; return (schemaOrType) => { @@ -1040,6 +1043,32 @@ export const getters = { }; }, + paginationHeadersFor(state, getters, rootState, rootGetters) { + return (schema) => { + const attributes = schema.attributes || {}; + const columns = attributes.columns || []; + + return state.paginationHeaders[schema.id]?.map((entry) => { + if ( typeof entry === 'string' ) { + const col = findBy(columns, 'name', entry); + + if ( col ) { + return { + ...fromSchema(col, rootGetters), // TODO: RC neater way of doing this + search: _rowValueGetter(col, false), + sort: [_rowValueGetter(col, false)], // Doesn't work at the moment + }; + } else { + return null; + } + } else { + return entry; + } + }) + .filter((col) => !!col); + }; + }, + headersFor(state, getters, rootState, rootGetters) { return (schema) => { const attributes = schema.attributes || {}; @@ -1096,43 +1125,6 @@ export const getters = { } return out; - - function fromSchema(col, rootGetters) { - let formatter, width, formatterOpts; - - if ( (col.format === '' || col.format === 'date') && col.name === 'Age' ) { - return AGE; - } - - if ( col.format === 'date' || col.type === 'date' ) { - formatter = 'Date'; - width = 120; - formatterOpts = { multiline: true }; - } - - if ( col.type === 'number' || col.type === 'int' ) { - formatter = 'Number'; - } - - const colName = col.name.includes(' ') ? col.name.split(' ').map((word) => word.charAt(0).toUpperCase() + word.substring(1) ).join('') : col.name; - - const exists = rootGetters['i18n/exists']; - const t = rootGetters['i18n/t']; - const labelKey = `tableHeaders.${ colName.charAt(0).toLowerCase() + colName.slice(1) }`; - const description = col.description || ''; - const tooltip = description && description[description.length - 1] === '.' ? description.slice(0, -1) : description; - - return { - name: col.name.toLowerCase(), - label: exists(labelKey) ? t(labelKey) : col.name, - value: _rowValueGetter(col), - sort: [col.field], - formatter, - formatterOpts, - width, - tooltip - }; - } }; }, @@ -1633,6 +1625,10 @@ export const mutations = { state.headers[type] = headers; }, + paginationHeaders(state, { type, paginationHeaders }) { + state.paginationHeaders[type] = paginationHeaders; + }, + hideBulkActions(state, { type, field }) { state.hideBulkActions[type] = field; }, @@ -1777,6 +1773,43 @@ export const actions = { } }; +function fromSchema(col, rootGetters) { + let formatter, width, formatterOpts; + + if ( (col.format === '' || col.format === 'date') && col.name === 'Age' ) { + return AGE; + } + + if ( col.format === 'date' || col.type === 'date' ) { + formatter = 'Date'; + width = 120; + formatterOpts = { multiline: true }; + } + + if ( col.type === 'number' || col.type === 'int' ) { + formatter = 'Number'; + } + + const colName = col.name.includes(' ') ? col.name.split(' ').map((word) => word.charAt(0).toUpperCase() + word.substring(1) ).join('') : col.name; + + const exists = rootGetters['i18n/exists']; + const t = rootGetters['i18n/t']; + const labelKey = `tableHeaders.${ colName.charAt(0).toLowerCase() + colName.slice(1) }`; + const description = col.description || ''; + const tooltip = description && description[description.length - 1] === '.' ? description.slice(0, -1) : description; + + return { + name: col.name.toLowerCase(), + label: exists(labelKey) ? t(labelKey) : col.name, + value: _rowValueGetter(col), + sort: [col.field], + formatter, + formatterOpts, + width, + tooltip + }; +} + function _sortGroup(tree, mode) { const by = ['weight:desc', 'namespaced', 'label']; @@ -1929,7 +1962,7 @@ function _findColumnByName(schema, colName) { return findBy(columns, 'name', colName); } -function _rowValueGetter(col) { +function _rowValueGetter(col, asFn = true) { // 'field' comes from the schema - typically it is of the form $.metadata.field[N] // We will use JsonPath to look up this value, which is costly - so if we can detect this format // Use a more efficient function to get the value @@ -1939,7 +1972,11 @@ function _rowValueGetter(col) { if (found && found.length === 2) { const fieldIndex = parseInt(found[1], 10); - return (row) => row.metadata?.fields?.[fieldIndex]; + if (asFn) { + return (row) => row.metadata?.fields?.[fieldIndex]; + } + + return `metadata.fields.${ fieldIndex }`; } return value; diff --git a/shell/utils/array.ts b/shell/utils/array.ts index c2b72e84b90..18fad611354 100644 --- a/shell/utils/array.ts +++ b/shell/utils/array.ts @@ -1,5 +1,5 @@ import xor from 'lodash/xor'; -import { get } from '@shell/utils/object'; +import { get, isEqual } from '@shell/utils/object'; export function removeObject(ary: T[], obj: T): T[] { const idx = ary.indexOf(obj); @@ -180,6 +180,25 @@ export function sameContents(aryA: T[], aryB: T[]): boolean { return xor(aryA, aryB).length === 0; } +export function sameArrayObjects(aryA: T[], aryB: T[]): boolean { + if (!aryA && !aryB) { + // catch calls from js (where props aren't type checked) + return false; + } + if (aryA?.length !== aryB?.length) { + // catch one null and not t'other, and different lengths + return false; + } + + for (let i = 0; i < aryA.length; i++) { + if (!isEqual(aryA[i], aryB[i])) { + return false; + } + } + + return true; +} + export function uniq(ary: T[]): T[] { const out: T[] = []; From 09614b01362e88d0c5fd188e02cdcbb03b976b7a Mon Sep 17 00:00:00 2001 From: Richard Cox Date: Fri, 9 Feb 2024 17:11:46 +0000 Subject: [PATCH 02/12] Server-side pagination Polish and fixes from alpha --- shell/assets/translations/en-us.yaml | 1 + shell/components/EventsTable.vue | 67 ----- shell/components/ResourceList/index.vue | 70 ++--- shell/components/ResourceTable.vue | 69 +++-- shell/components/SortableTable/grouping.js | 9 +- shell/components/SortableTable/index.vue | 42 ++- shell/components/SortableTable/paging.js | 2 - shell/components/SortableTable/selection.js | 3 +- shell/components/SortableTable/sorting.js | 28 +- shell/components/nav/NamespaceFilter.vue | 10 + shell/config/pagination-table-headers.js | 96 ++----- shell/config/product/explorer.js | 44 ++-- shell/mixins/resource-fetch-api-pagination.js | 244 +++++++++++------- shell/mixins/resource-fetch.js | 26 +- shell/models/schema.js | 3 + shell/pages/c/_cluster/apps/charts/index.vue | 1 + shell/plugins/dashboard-store/actions.js | 15 +- shell/plugins/dashboard-store/getters.js | 8 +- shell/plugins/dashboard-store/mutations.js | 59 ++++- ....ts => accept-or-reject-socket-message.ts} | 32 ++- shell/plugins/steve/getters.js | 61 ++++- shell/plugins/steve/mutations.js | 3 + .../projectAndNamespaceFiltering.utils.ts | 4 +- shell/plugins/steve/schema.d.ts | 13 + ...ion-utils.ts => steve-pagination-utils.ts} | 48 ++-- shell/plugins/steve/subscribe.js | 10 +- shell/store/type-map.js | 194 ++++---------- shell/store/type-map.utils.ts | 183 +++++++++++++ shell/types/resources/settings.d.ts | 32 +++ .../{ => resources}/userPreferences.d.ts | 1 - .../store}/dashboard-store.types.ts | 9 +- shell/types/store/type-map.d.ts | 15 ++ shell/types/store/vuex.d.ts | 9 + shell/utils/group.js | 70 ----- shell/utils/pagination-utils.ts | 106 ++++++++ 35 files changed, 974 insertions(+), 613 deletions(-) delete mode 100644 shell/components/EventsTable.vue rename shell/plugins/steve/{subscribe-namespace-handler.ts => accept-or-reject-socket-message.ts} (64%) create mode 100644 shell/plugins/steve/schema.d.ts rename shell/plugins/steve/{pagination-utils.ts => steve-pagination-utils.ts} (67%) create mode 100644 shell/store/type-map.utils.ts create mode 100644 shell/types/resources/settings.d.ts rename shell/types/{ => resources}/userPreferences.d.ts (87%) rename shell/{plugins/dashboard-store => types/store}/dashboard-store.types.ts (70%) create mode 100644 shell/types/store/type-map.d.ts create mode 100644 shell/types/store/vuex.d.ts delete mode 100644 shell/utils/group.js create mode 100644 shell/utils/pagination-utils.ts diff --git a/shell/assets/translations/en-us.yaml b/shell/assets/translations/en-us.yaml index 823982e1f97..64801654fc3 100644 --- a/shell/assets/translations/en-us.yaml +++ b/shell/assets/translations/en-us.yaml @@ -4781,6 +4781,7 @@ resourceList: createFromYaml: Create from YAML createResource: "Create {resourceName}" nsFiltering: "Please select one or more namespaces or projects using the filter above." + nsFilteringGeneric: "Please select a valid namespace or project option using the filter above." nsFilterToolTip: "Filtering is restricted to projects and namespaces" resourceLoadingIndicator: loading: Loading diff --git a/shell/components/EventsTable.vue b/shell/components/EventsTable.vue deleted file mode 100644 index 56d4efa9a2e..00000000000 --- a/shell/components/EventsTable.vue +++ /dev/null @@ -1,67 +0,0 @@ - - - diff --git a/shell/components/ResourceList/index.vue b/shell/components/ResourceList/index.vue index a7869e2470e..a7841ec0850 100644 --- a/shell/components/ResourceList/index.vue +++ b/shell/components/ResourceList/index.vue @@ -8,8 +8,7 @@ import IconMessage from '@shell/components/IconMessage.vue'; import { ResourceListComponentName } from './resource-list.config'; import { PanelLocation, ExtensionPoint } from '@shell/core/types'; import ExtensionPanel from '@shell/components/ExtensionPanel'; -import { sameArrayObjects, sameContents } from '@shell/utils/array'; -import { isEqual } from '@shell/utils/object'; +import { sameContents } from '@shell/utils/array'; export default { name: ResourceListComponentName, @@ -56,7 +55,7 @@ export default { // If your list page has a fetch then it's responsible for populating rows itself if ( component?.fetch ) { - this.hasFetch = true; + this.componentWillFetch = true; } // If the custom component supports it, ask it what resources it loads, so we can @@ -69,16 +68,15 @@ export default { } } - if ( !this.hasFetch ) { + if ( !this.componentWillFetch ) { if ( !schema ) { store.dispatch('loadingError', new Error(this.t('nav.failWhale.resourceListNotFound', { resource }, true))); return; } - // See comment for `namespaceFilterRequired` watcher, skip fetch if we don't have a valid NS - // TODO: RC TODO tie in to comment above - if (!this.namespaceFilterRequired && !this.canPaginate) { + // See comment for `namespaceFilter` and `pagination` watchers, skip fetch if we're not ready yet... and something is going to call fetch later on + if (!this.namespaceFilterRequired && (!this.canPaginate || this.refreshFlag)) { await this.$fetchType(resource); } } @@ -105,7 +103,11 @@ export default { extensionType: ExtensionPoint.PANEL, extensionLocation: PanelLocation.RESOURCE_LIST, loadResources: [resource], // List of resources that will be loaded, this could be many (`Workloads`) - hasFetch: false, + /** + * Will the custom component handle the fetch of resources.... + * or will this instance fetch resources + */ + componentWillFetch: false, // manual refresh manualRefreshInit: false, watch: false, @@ -115,7 +117,7 @@ export default { // incremental loading loadIndeterminate: false, // query param for simple filtering - useQueryParamsForSimpleFiltering: true + useQueryParamsForSimpleFiltering: true, }; }, @@ -126,11 +128,7 @@ export default { return []; } - if (this.pagination) { - return this.paginationHeaders; - } - - return this.$store.getters['type-map/headersFor'](this.schema); + return this.$store.getters['type-map/headersFor'](this.schema, this.canPaginate); }, groupBy() { @@ -144,6 +142,7 @@ export default { }, watch: { + /** * When a NS filter is required and the user selects a different one, kick off a new set of API requests * @@ -154,29 +153,26 @@ export default { * This covers case 1 */ namespaceFilter(neu, old) { - if (sameContents(neu, old)) { - return; - } + if (neu && !this.componentWillFetch) { + if (sameContents(neu, old)) { + return; + } - if (neu && !this.hasFetch) { this.$fetchType(this.resource); } }, + /** + * When a pagination is required and the user changes page / sort / filter, kick off a new set of API requests + * + * ResourceList has two modes + * 1) ResourceList component handles API request to fetch resources + * 2) Custom list component handles API request to fetch resources + * + * This covers case 1 + */ pagination(neu, old) { - const { filter: neuFilter, sort: neuSort, ...newPrimitiveTypes } = neu; - const { filter: oldFilter, sort: oldSort, ...oldPrimitiveTypes } = old; - - if ( - isEqual(newPrimitiveTypes, oldPrimitiveTypes) && - isEqual(neuFilter, oldFilter) && - sameArrayObjects(neuSort, oldSort) - ) { - // TODO: RC TEST more - return; - } - - if (neu && !this.hasFetch) { + if (neu && !this.componentWillFetch && this.paginationEqual(neu, old)) { this.$fetchType(this.resource); } }, @@ -208,6 +204,16 @@ export default { {{ t('resourceList.nsFiltering') }} + + +
diff --git a/shell/components/ResourceTable.vue b/shell/components/ResourceTable.vue index f8831eb85fb..29fa4d18048 100644 --- a/shell/components/ResourceTable.vue +++ b/shell/components/ResourceTable.vue @@ -194,25 +194,31 @@ export default { }, data() { - const options = this.$store.getters[`type-map/optionsFor`](this.schema); - const listGroups = options?.listGroups || []; - const listGroupsWillOverride = options?.listGroupsWillOverride; - const listGroupMapped = listGroups.reduce((acc, grp) => { - acc[grp.value] = grp; - - return acc; - }, {}); - // Confirm which store we're in, if schema isn't available we're probably showing a list with different types const inStore = this.schema?.id ? this.$store.getters['currentStore'](this.schema.id) : undefined; - return { - listGroups, listGroupsWillOverride, listGroupMapped, inStore - }; + return { inStore }; }, computed: { + options() { + return this.$store.getters[`type-map/optionsFor`](this.schema, !!this.externalPagination); + }, + + _listGroupMapped() { + return this.options?.listGroups?.reduce((acc, grp) => { + acc[grp.value] = grp; + + return acc; + }, {}); + }, + + _mandatorySort() { + return this.options?.listMandatorySort; + }, + ...mapGetters(['currentProduct']), + isNamespaced() { if ( this.namespaced !== null ) { return this.namespaced; @@ -247,7 +253,7 @@ export default { if ( this.headers ) { headers = this.headers.slice(); } else { - headers = this.$store.getters['type-map/headersFor'](this.schema); // TODO: RC not required atm, but needs to be integrated with pagination headers + headers = this.$store.getters['type-map/headersFor'](this.schema, !!this.externalPagination); } // add custom table columns provided by the extensions ExtensionPoint.TABLE_COL hook @@ -294,7 +300,7 @@ export default { } // If we are grouping by a custom group, it may specify that we hide a specific column - const custom = this.listGroupMapped[this.group]; + const custom = this._listGroupMapped?.[this.group]; if (custom?.hideColumn) { const idx = headers.findIndex((header) => header.name === custom.hideColumn); @@ -359,7 +365,12 @@ export default { const exists = this.groupOptions.find((g) => g.value === this._group); if (!exists) { - return DEFAULT_GROUP; + // Attempt to find the default option in available options... + // if not use the first value in the options collection... + // and if not that just fall back to the default + const defaultGroup = this.groupOptions.find((g) => g.value === DEFAULT_GROUP); + + return defaultGroup ? DEFAULT_GROUP : this.groupOptions[0]?.value || DEFAULT_GROUP; } return this._group; @@ -372,7 +383,7 @@ export default { showGrouping() { if ( this.groupable === null ) { const namespaceGroupable = this.$store.getters['isMultipleNamespaces'] && this.isNamespaced; - const customGroupable = this.listGroups.length > 0; + const customGroupable = !!this.options?.listGroups?.length; return namespaceGroupable || customGroupable; } @@ -382,16 +393,19 @@ export default { computedGroupBy() { if ( this.groupBy ) { + // This probably comes from the type-map config for the resource (see ResourceList) return this.groupBy; } if ( this.group === 'namespace' && this.showGrouping ) { + // This switches to group rows by a key which is the label for the group (??) return 'groupByLabel'; } - const custom = this.listGroupMapped[this.group]; + const custom = this._listGroupMapped?.[this.group]; - if (custom && custom.field) { + if (custom?.field) { + // Override the normal filtering return custom.field; } @@ -399,8 +413,10 @@ export default { }, groupOptions() { - if (this.listGroupsWillOverride && this.listGroups?.length) { - return this.listGroups; + // Ignore the defaults below, we have an override set of groups + // REPLACE (instead of SUPPLEMENT) defaults with listGroups (given listGroupsWillOverride is true) + if (this.options?.listGroupsWillOverride && !!this.options?.listGroups?.length) { + return this.options?.listGroups; } const standard = [ @@ -416,7 +432,12 @@ export default { }, ]; - return standard.concat(this.listGroups); + // SUPPLEMENT (instead of REPLACE) defaults with listGroups (given listGroupsWillOverride is false) + if (!!this.options?.listGroups?.length) { + return standard.concat(this.options.listGroups); + } + + return standard; }, parsedPagingParams() { @@ -436,6 +457,7 @@ export default { pluralLabel: this.$store.getters['type-map/labelFor'](this.schema, 99), }; }, + }, methods: { @@ -497,7 +519,7 @@ export default { this.keyAction('detail'); } } - } + }, }; @@ -529,8 +551,10 @@ export default { :force-update-live-and-delayed="forceUpdateLiveAndDelayed" :external-pagination="externalPagination" :external-pagination-result="externalPaginationResult" + :mandatory-sort="_mandatorySort" @clickedActionButton="handleActionButtonClick" @group-value-change="group = $event" + v-on="$listeners" >