Feat: entry sorting (#3494)

* refactor: typescript search actions, add tests avoid duplicate search

* refactor: switch from promise chain to async/await in loadEntries

* feat: add sorting, initial commit

* fix: set isFetching to true on entries request

* fix: ui improvments and bug fixes

* test: fix tests

* feat(backend-gitlab): cache local tree)

* fix: fix prop type warning

* refactor: code cleanup

* feat(backend-bitbucket): add local tree caching support

* feat: swtich to orderBy and support multiple sort keys

* fix: backoff function

* fix: improve backoff

* feat: infer sortable fields

* feat: fetch file commit metadata - initial commit

* feat: extract file author and date, finalize GitLab & Bitbucket

* refactor: code cleanup

* feat: handle github rate limit errors

* refactor: code cleanup

* fix(github): add missing author and date when traversing cursor

* fix: add missing author and date when traversing cursor

* refactor: code cleanup

* refactor: code cleanup

* refactor: code cleanup

* test: fix tests

* fix: rebuild local tree when head doesn't exist in remote branch

* fix: allow sortable fields to be an empty array

* fix: allow translation of built in sort fields

* build: fix proxy server build

* fix: hide commit author and date fields by default on non git backends

* fix(algolia): add listAllEntries method for alogolia integration

* fix: handle sort fields overflow

* test(bitbucket): re-record some bitbucket e2e tests

* test(bitbucket): fix media library test

* refactor(gitgateway-gitlab): share request code and handle 404 errors

* fix: always show commit date by default

* docs: add sortableFields

* refactor: code cleanup

* improvement: drop multi-sort, rework sort UI

* chore: force main package bumps

Co-authored-by: Shawn Erquhart <shawn@erquh.art>

packages/netlify-cms-core/src/reducers/collections.ts

@@ -3,7 +3,7 @@ import { get, escapeRegExp } from 'lodash';
 import consoleError from '../lib/consoleError';
 import { CONFIG_SUCCESS } from '../actions/config';
 import { FILES, FOLDER } from '../constants/collectionTypes';
-import { INFERABLE_FIELDS, IDENTIFIER_FIELDS } from '../constants/fieldInference';
+import { INFERABLE_FIELDS, IDENTIFIER_FIELDS, SORTABLE_FIELDS } from '../constants/fieldInference';
 import { formatExtensions } from '../formats/formats';
 import {
   CollectionsAction,
@@ -15,6 +15,7 @@ import {
 } from '../types/redux';
 import { selectMediaFolder } from './entries';
 import { keyToPathArray } from '../lib/stringTemplate';
+import { Backend } from '../backend';
 
 const collections = (state = null, action: CollectionsAction) => {
   switch (action.type) {
@@ -288,6 +289,7 @@ export const selectIdentifier = (collection: Collection) => {
     fieldNames.find(name => name?.toLowerCase().trim() === id.toLowerCase().trim()),
   );
 };
+
 export const selectInferedField = (collection: Collection, fieldName: string) => {
   if (fieldName === 'title' && collection.get('identifier_field')) {
     return selectIdentifier(collection);
@@ -337,4 +339,56 @@ export const selectInferedField = (collection: Collection, fieldName: string) =>
   return null;
 };
 
+export const COMMIT_AUTHOR = 'commit_author';
+export const COMMIT_DATE = 'commit_date';
+
+export const selectDefaultSortableFields = (collection: Collection, backend: Backend) => {
+  let defaultSortable = SORTABLE_FIELDS.map((type: string) => {
+    const field = selectInferedField(collection, type);
+    if (backend.isGitBackend() && type === 'author' && !field) {
+      // default to commit author if not author field is found
+      return COMMIT_AUTHOR;
+    }
+    return field;
+  }).filter(Boolean);
+
+  if (backend.isGitBackend()) {
+    // always have commit date by default
+    defaultSortable = [COMMIT_DATE, ...defaultSortable];
+  }
+
+  return defaultSortable as string[];
+};
+
+export const selectSortableFields = (collection: Collection, t: (key: string) => string) => {
+  const fields = collection
+    .get('sortableFields')
+    .toArray()
+    .map(key => {
+      if (key === COMMIT_DATE) {
+        return { key, field: { name: key, label: t('collection.defaultFields.updatedOn.label') } };
+      }
+      const field = selectField(collection, key);
+      if (key === COMMIT_AUTHOR && !field) {
+        return { key, field: { name: key, label: t('collection.defaultFields.author.label') } };
+      }
+
+      return { key, field: field?.toJS() };
+    })
+    .filter(item => !!item.field)
+    .map(item => ({ ...item.field, key: item.key }));
+
+  return fields;
+};
+
+export const selectSortDataPath = (collection: Collection, key: string) => {
+  if (key === COMMIT_DATE) {
+    return 'updatedOn';
+  } else if (key === COMMIT_AUTHOR && !selectField(collection, key)) {
+    return 'author';
+  } else {
+    return `data.${key}`;
+  }
+};
+
 export default collections;

packages/netlify-cms-core/src/reducers/cursors.js

@@ -1,6 +1,6 @@
 import { fromJS } from 'immutable';
 import { Cursor } from 'netlify-cms-lib-util';
-import { ENTRIES_SUCCESS } from 'Actions/entries';
+import { ENTRIES_SUCCESS, SORT_ENTRIES_SUCCESS } from 'Actions/entries';
 
 // Since pagination can be used for a variety of views (collections
 // and searches are the most common examples), we namespace cursors by
@@ -16,7 +16,9 @@ const cursors = (state = fromJS({ cursorsByType: { collectionEntries: {} } }), a
         Cursor.create(action.payload.cursor).store,
       );
     }
-
+    case SORT_ENTRIES_SUCCESS: {
+      return state.deleteIn(['cursorsByType', 'collectionEntries', action.payload.collection]);
+    }
     default:
       return state;
   }

packages/netlify-cms-core/src/reducers/entries.ts

@@ -1,4 +1,4 @@
-import { Map, List, fromJS } from 'immutable';
+import { Map, List, fromJS, OrderedMap } from 'immutable';
 import { dirname, join } from 'path';
 import {
   ENTRY_REQUEST,
@@ -8,6 +8,9 @@ import {
   ENTRIES_SUCCESS,
   ENTRIES_FAILURE,
   ENTRY_DELETE_SUCCESS,
+  SORT_ENTRIES_REQUEST,
+  SORT_ENTRIES_SUCCESS,
+  SORT_ENTRIES_FAILURE,
 } from '../actions/entries';
 import { SEARCH_ENTRIES_SUCCESS } from '../actions/search';
 import {
@@ -26,10 +29,17 @@ import {
   EntryMap,
   EntryField,
   CollectionFiles,
+  EntriesSortRequestPayload,
+  EntriesSortSuccessPayload,
+  EntriesSortFailurePayload,
+  SortMap,
+  SortObject,
+  Sort,
+  SortDirection,
 } from '../types/redux';
 import { folderFormatter } from '../lib/formatters';
 import { isAbsolutePath, basename } from 'netlify-cms-lib-util';
-import { trim } from 'lodash';
+import { trim, once, sortBy, set } from 'lodash';
 
 let collection: string;
 let loadedEntries: EntryObject[];
@@ -37,7 +47,60 @@ let append: boolean;
 let page: number;
 let slug: string;
 
-const entries = (state = Map({ entities: Map(), pages: Map() }), action: EntriesAction) => {
+const storageSortKey = 'netlify-cms.entries.sort';
+type StorageSortObject = SortObject & { index: number };
+type StorageSort = { [collection: string]: { [key: string]: StorageSortObject } };
+
+const loadSort = once(() => {
+  const sortString = localStorage.getItem(storageSortKey);
+  if (sortString) {
+    try {
+      const sort: StorageSort = JSON.parse(sortString);
+      let map = Map() as Sort;
+      Object.entries(sort).forEach(([collection, sort]) => {
+        let orderedMap = OrderedMap() as SortMap;
+        sortBy(Object.values(sort), ['index']).forEach(value => {
+          const { key, direction } = value;
+          orderedMap = orderedMap.set(key, fromJS({ key, direction }));
+        });
+        map = map.set(collection, orderedMap);
+      });
+      return map;
+    } catch (e) {
+      return Map() as Sort;
+    }
+  }
+  return Map() as Sort;
+});
+
+const clearSort = () => {
+  localStorage.removeItem(storageSortKey);
+};
+
+const persistSort = (sort: Sort | undefined) => {
+  if (sort) {
+    const storageSort: StorageSort = {};
+    sort.keySeq().forEach(key => {
+      const collection = key as string;
+      const sortObjects = (sort
+        .get(collection)
+        .valueSeq()
+        .toJS() as SortObject[]).map((value, index) => ({ ...value, index }));
+
+      sortObjects.forEach(value => {
+        set(storageSort, [collection, value.key], value);
+      });
+    });
+    localStorage.setItem(storageSortKey, JSON.stringify(storageSort));
+  } else {
+    clearSort();
+  }
+};
+
+const entries = (
+  state = Map({ entities: Map(), pages: Map(), sort: loadSort() }),
+  action: EntriesAction,
+) => {
   switch (action.type) {
     case ENTRY_REQUEST: {
       const payload = action.payload as EntryRequestPayload;
@@ -59,7 +122,13 @@ const entries = (state = Map({ entities: Map(), pages: Map() }), action: Entries
 
     case ENTRIES_REQUEST: {
       const payload = action.payload as EntriesRequestPayload;
-      return state.setIn(['pages', payload.collection, 'isFetching'], true);
+      const newState = state.withMutations(map => {
+        map.deleteIn(['sort', payload.collection]);
+        map.setIn(['pages', payload.collection, 'isFetching'], true);
+      });
+
+      clearSort();
+      return newState;
     }
 
     case ENTRIES_SUCCESS: {
@@ -123,11 +192,74 @@ const entries = (state = Map({ entities: Map(), pages: Map() }), action: Entries
       });
     }
 
+    case SORT_ENTRIES_REQUEST: {
+      const payload = action.payload as EntriesSortRequestPayload;
+      const { collection, key, direction } = payload;
+      const newState = state.withMutations(map => {
+        const sort = OrderedMap({ [key]: Map({ key, direction }) });
+        map.setIn(['sort', collection], sort);
+        map.setIn(['pages', collection, 'isFetching'], true);
+        map.deleteIn(['pages', collection, 'page']);
+      });
+      persistSort(newState.get('sort') as Sort);
+      return newState;
+    }
+
+    case SORT_ENTRIES_SUCCESS: {
+      const payload = action.payload as EntriesSortSuccessPayload;
+      const { collection, entries } = payload;
+      loadedEntries = entries;
+      const newState = state.withMutations(map => {
+        loadedEntries.forEach(entry =>
+          map.setIn(
+            ['entities', `${entry.collection}.${entry.slug}`],
+            fromJS(entry).set('isFetching', false),
+          ),
+        );
+        map.setIn(['pages', collection, 'isFetching'], false);
+        const ids = List(loadedEntries.map(entry => entry.slug));
+        map.setIn(
+          ['pages', collection],
+          Map({
+            page: 1,
+            ids,
+          }),
+        );
+      });
+      return newState;
+    }
+
+    case SORT_ENTRIES_FAILURE: {
+      const payload = action.payload as EntriesSortFailurePayload;
+      const { collection, key } = payload;
+      const newState = state.withMutations(map => {
+        map.deleteIn(['sort', collection, key]);
+        map.setIn(['pages', collection, 'isFetching'], false);
+      });
+      persistSort(newState.get('sort') as Sort);
+      return newState;
+    }
+
     default:
       return state;
   }
 };
 
+export const selectEntriesSort = (entries: Entries, collection: string) => {
+  const sort = entries.get('sort') as Sort | undefined;
+  return sort?.get(collection);
+};
+
+export const selectEntriesSortFields = (entries: Entries, collection: string) => {
+  const sort = selectEntriesSort(entries, collection);
+  const values =
+    sort
+      ?.valueSeq()
+      .filter(v => v?.get('direction') !== SortDirection.None)
+      .toArray() || [];
+  return values;
+};
+
 export const selectEntry = (state: Entries, collection: string, slug: string) =>
   state.getIn(['entities', `${collection}.${slug}`]);
 
@@ -136,7 +268,18 @@ export const selectPublishedSlugs = (state: Entries, collection: string) =>
 
 export const selectEntries = (state: Entries, collection: string) => {
   const slugs = selectPublishedSlugs(state, collection);
-  return slugs && slugs.map(slug => selectEntry(state, collection, slug as string));
+  const entries =
+    slugs && (slugs.map(slug => selectEntry(state, collection, slug as string)) as List<EntryMap>);
+
+  return entries;
+};
+
+export const selectEntriesLoaded = (state: Entries, collection: string) => {
+  return !!state.getIn(['pages', collection]);
+};
+
+export const selectIsFetching = (state: Entries, collection: string) => {
+  return state.getIn(['pages', collection, 'isFetching'], false);
 };
 
 const DRAFT_MEDIA_FILES = 'DRAFT_MEDIA_FILES';

packages/netlify-cms-core/src/reducers/search.js

@@ -31,6 +31,7 @@ const entries = (state = defaultState, action) => {
         return state.withMutations(map => {
           map.set('isFetching', true);
           map.set('term', action.payload.searchTerm);
+          map.set('page', action.payload.page);
         });
       }
       return state;