Improve the comprehensive pagination engine

- Add documentation
- Rename things for clarity
- Add support for transforming the data, besides filtering

Author: csillag

.changelog/1385.internal.md

@@ -0,0 +1 @@
+Improve the comprehensive pagination engine

src/app/components/Table/PaginationEngine.ts

@@ -7,15 +7,67 @@ export interface SimplePaginationEngine {
   linkToPage: (pageNumber: number) => To
 }
 
-export interface PaginatedResults<Item> {
+/**
+ * The data returned by a comprehensive pagination engine to the data consumer component
+ */
+export interface ComprehensivePaginatedResults<Item, ExtractedData = typeof undefined> {
+  /**
+   * Control interface that can be plugged to a Table's `pagination` prop
+   */
   tablePaginationProps: TablePaginationProps
+
+  /**
+   * The data provided to the data consumer in the current window
+   */
   data: Item[] | undefined
+
+  /**
+   * Any extra data produced by the transformer function (besides the array of items)
+   */
+  extractedData?: ExtractedData | undefined
+
+  /**
+   * Is the data set still loading from the server?
+   */
+  isLoading: boolean
+
+  /**
+   * Has the data been loaded from the server?
+   */
+  isFetched: boolean
 }
 
-export interface ComprehensivePaginationEngine<Item, QueryResult extends List> {
-  selectedPage: number
-  offsetForQuery: number
-  limitForQuery: number
-  paramsForQuery: { offset: number; limit: number }
-  getResults: (queryResult: QueryResult | undefined, key?: keyof QueryResult) => PaginatedResults<Item>
+/**
+ * A Comprehensive PaginationEngine sits between the server and the consumer of the data and does transformations
+ *
+ * Specifically, the interface for loading the data and the one for the data consumers are decoupled.
+ */
+export interface ComprehensivePaginationEngine<
+  Item,
+  QueryResult extends List,
+  ExtractedData = typeof undefined,
+> {
+  /**
+   * The currently selected page from the data consumer's POV
+   */
+  selectedPageForClient: number
+
+  /**
+   * Parameters for data to be loaded from the server
+   */
+  paramsForServer: { offset: number; limit: number }
+
+  /**
+   * Get the current data/state info for the data consumer component.
+   *
+   * @param isLoading Is the data still being loaded from the server?
+   * @param queryResult the data coming in the server, requested according to this engine's specs, including metadata
+   * @param key The field where the actual records can be found within queryResults
+   */
+  getResults: (
+    isLoading: boolean,
+    isFetched: boolean,
+    queryResult: QueryResult | undefined,
+    key?: keyof QueryResult,
+  ) => ComprehensivePaginatedResults<Item, ExtractedData>
 }

src/app/components/Table/useClientSidePagination.ts

@@ -1,14 +1,42 @@
 import { To, useSearchParams } from 'react-router-dom'
 import { AppErrors } from '../../../types/errors'
-import { ComprehensivePaginationEngine } from './PaginationEngine'
+import { ComprehensivePaginatedResults, ComprehensivePaginationEngine } from './PaginationEngine'
 import { List } from '../../../oasis-nexus/api'
 import { TablePaginationProps } from './TablePagination'
 
-type ClientSizePaginationParams<Item> = {
+type ClientSizePaginationParams<Item, QueryResult extends List, ExtractedData = typeof undefined> = {
+  /**
+   * How should we call the query parameter (in the URL)?
+   */
   paramName: string
+
+  /**
+   * The pagination page size from the POV of the data consumer component
+   */
   clientPageSize: number
+
+  /**
+   * The pagination page size used for actually loading the data from the server.
+   *
+   * Please note that currently this engine doesn't handle when the data consumer requires data which is not
+   * part of the initial window on the server side.
+   */
   serverPageSize: number
+
+  /**
+   * Filter to be applied to the loaded data.
+   *
+   * If both transform and filter is set, transform will run first.
+   */
   filter?: (item: Item) => boolean
+
+  /**
+   * Transformation to be applied after loading the data from the server, before presenting it to the data consumer component
+   *
+   * Can be used for ordering, aggregation, etc.D
+   * If both transform and filter is set, transform will run first.
+   */
+  transform?: (input: Item[], results: QueryResult) => [Item[], ExtractedData]
 }
 
 const knownListKeys: string[] = ['total_count', 'is_total_count_clipped']
@@ -27,13 +55,20 @@ function findListIn<T extends List, Item>(data: T): Item[] {
   }
 }
 
-export function useClientSidePagination<Item, QueryResult extends List>({
+/**
+ * The ClientSidePagination engine loads the data from the server with a big window in one go, for in-memory pagination
+ */
+export function useClientSidePagination<Item, QueryResult extends List, ExtractedData = typeof undefined>({
   paramName,
   clientPageSize,
   serverPageSize,
   filter,
-}: ClientSizePaginationParams<Item>): ComprehensivePaginationEngine<Item, QueryResult> {
-  const selectedServerPage = 1
+  transform,
+}: ClientSizePaginationParams<Item, QueryResult, ExtractedData>): ComprehensivePaginationEngine<
+  Item,
+  QueryResult,
+  ExtractedData
+> {
   const [searchParams] = useSearchParams()
   const selectedClientPageString = searchParams.get(paramName)
   const selectedClientPage = parseInt(selectedClientPageString ?? '1', 10)
@@ -57,30 +92,43 @@ export function useClientSidePagination<Item, QueryResult extends List>({
     return { search: newSearchParams.toString() }
   }
 
-  const limit = serverPageSize
-  const offset = (selectedServerPage - 1) * clientPageSize
+  // From the server, we always want to load the first batch of data, with the provided (big) window.
+  // In theory, we could move this window as required, but currently this is not implemented.
+  const selectedServerPage = 1
+
+  // The query parameters that should be used for loading the data from the server
   const paramsForQuery = {
-    offset,
-    limit,
+    offset: (selectedServerPage - 1) * serverPageSize,
+    limit: serverPageSize,
   }
 
   return {
-    selectedPage: selectedClientPage,
-    offsetForQuery: offset,
-    limitForQuery: limit,
-    paramsForQuery,
-    getResults: (queryResult, key) => {
-      const data = queryResult
-        ? key
-          ? (queryResult[key] as Item[])
-          : findListIn<QueryResult, Item>(queryResult)
+    selectedPageForClient: selectedClientPage,
+    paramsForServer: paramsForQuery,
+    getResults: (
+      isLoading,
+      isFetched,
+      queryResult,
+      key,
+    ): ComprehensivePaginatedResults<Item, ExtractedData> => {
+      const data = queryResult // we want to get list of items out from the incoming results
+        ? key // do we know where (in which field) to look?
+          ? (queryResult[key] as Item[]) // If yes, just get out the data
+          : findListIn<QueryResult, Item>(queryResult) // If no, we will try to guess
         : undefined
-      const filteredData = !!data && !!filter ? data.filter(filter) : data
 
+      // Apply the specified client-side transformation
+      const [transformedData, extractedData] = !!data && !!transform ? transform(data, queryResult!) : [data]
+
+      // Apply the specified filtering
+      const filteredData = !!transformedData && !!filter ? transformedData.filter(filter) : transformedData
+
+      // The data window from the POV of the data consumer component
       const offset = (selectedClientPage - 1) * clientPageSize
       const limit = clientPageSize
       const dataWindow = filteredData ? filteredData.slice(offset, offset + limit) : undefined
 
+      // The control interface for the data consumer component (i.e. Table)
       const tableProps: TablePaginationProps = {
         selectedPage: selectedClientPage,
         linkToPage,
@@ -96,8 +144,10 @@ export function useClientSidePagination<Item, QueryResult extends List>({
       return {
         tablePaginationProps: tableProps,
         data: dataWindow,
+        extractedData,
+        isLoading,
+        isFetched,
       }
     },
-    // tableProps,
   }
 }

src/app/components/Table/useComprehensiveSearchParamsPagination.ts

@@ -64,11 +64,9 @@ export function useComprehensiveSearchParamsPagination<Item, QueryResult extends
   }
 
   return {
-    selectedPage,
-    offsetForQuery: offset,
-    limitForQuery: limit,
-    paramsForQuery,
-    getResults: (queryResult, key) => {
+    selectedPageForClient: selectedPage,
+    paramsForServer: paramsForQuery,
+    getResults: (isLoading, isFetched, queryResult, key) => {
       const data = queryResult
         ? key
           ? (queryResult[key] as Item[])
@@ -85,6 +83,8 @@ export function useComprehensiveSearchParamsPagination<Item, QueryResult extends
       return {
         tablePaginationProps: tableProps,
         data: filteredData,
+        isLoading,
+        isFetched,
       }
     },
     // tableProps,

src/app/pages/TokenDashboardPage/hook.ts

@@ -72,7 +72,7 @@ export const _useTokenTransfers = (scope: SearchScope, params: undefined | GetRu
     network,
     layer, // This is OK since consensus has been handled separately
     {
-      ...pagination.paramsForQuery,
+      ...pagination.paramsForServer,
       type: RuntimeEventType.evmlog,
       // The following is the hex-encoded signature for Transfer(address,address,uint256)
       evm_log_signature: 'ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
@@ -87,16 +87,16 @@ export const _useTokenTransfers = (scope: SearchScope, params: undefined | GetRu
 
   const { isFetched, isLoading, data } = query
 
-  const results = pagination.getResults(data?.data)
+  const results = pagination.getResults(isLoading, isFetched, data?.data)
 
-  if (isFetched && pagination.selectedPage > 1 && !results.data?.length) {
+  if (isFetched && pagination.selectedPageForClient > 1 && !results.data?.length) {
     throw AppErrors.PageDoesNotExist
   }
 
   return {
     isLoading,
     isFetched,
-    results: pagination.getResults(data?.data),
+    results,
   }
 }