Rebase of task-force from Orchid release branch.

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change history for ui-data-import
 
-## **6.1.0** (in progress)
+## **6.0.10** (in progress)
+* Implement file upload to S3 (UIDATIMP-1460)
+* Perform rough split on front-end for multipart Upload (UIDATIMP-1468)
+* Notify users that large files will be split (UIDATIMP-1463)
+* Retrieve backend configuration for splitting in UI (UIDATIMP-1462)
+* Render details of composite jobs in on consolidated running job cards (UIDATIMP-1466)
+* Display a link to download a slice from the automated splitting process (UIDATIMP-1510)
+* Cancel upload/running a split job (UIDATIMP-1469)
 
 ### Features added:
 * Landing page: Create hotlink from job profile name in log to the job profile details (UIDATIMP-1355)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@folio/data-import",
-  "version": "6.0.9",
+  "version": "6.0.10",
   "description": "Data Import manager",
   "main": "src/index.js",
   "repository": "folio-org/ui-data-import",
@@ -21,6 +21,9 @@
     "build-mod-descriptor": "stripes mod descriptor --full --strict | jq '.[]' > module-descriptor.json ",
     "formatjs-compile": "formatjs compile-folder --ast --format simple ./translations/ui-data-import ./translations/ui-data-import/compiled"
   },
+  "jest": {
+    "testEnvironment": "node"
+  },
   "devDependencies": {
     "@babel/core": "^7.17.12",
     "@babel/eslint-parser": "^7.17.0",
@@ -92,7 +95,8 @@
     "redux": "^4.0.5"
   },
   "resolutions": {
-    "moment": "~2.29.4"
+    "moment": "~2.29.4",
+    "@rehooks/local-storage": "2.4.4"
   },
   "optionalDependencies": {
     "@folio/plugin-find-organization": "^4.0.0"
@@ -182,7 +186,7 @@
       }
     ],
     "okapiInterfaces": {
-      "data-import": "3.0",
+      "data-import": "3.1",
       "source-manager-job-executions": "3.0",
       "data-import-converter-storage": "1.2"
     },

src/components/DataFetcher/DataFetcher.js

@@ -19,6 +19,7 @@ import {
 } from '../../utils';
 
 import { DataFetcherContext } from '.';
+import { requestConfiguration } from '../../utils/multipartUpload';
 
 const {
   RUNNING,
@@ -56,21 +57,42 @@ const logsUrlParams = [
 const jobsUrl = createUrlFromArray('metadata-provider/jobExecutions', jobsUrlParams);
 const logsUrl = createUrlFromArray('metadata-provider/jobExecutions', logsUrlParams);
 
+const compositeJobsUrl = createUrlFromArray('metadata-provider/jobExecutions', [...jobsUrlParams, 'subordinationTypeNotAny=COMPOSITE_CHILD']);
+const compositeLogsUrl = createUrlFromArray('metadata-provider/jobExecutions', [...logsUrlParams, 'subordinationTypeNotAny=COMPOSITE_PARENT']);
+
+export function getJobSplittingURL(resources, splittingURL, nonSplitting) {
+  const { split_status: splitStatus } = resources;
+  if (!splitStatus?.isPending) {
+    if (splitStatus?.records[0]?.splitStatus) {
+      return splittingURL;
+    } else if (splitStatus?.records[0]?.splitStatus === false) {
+      return nonSplitting;
+    }
+  }
+  return undefined;
+}
+
 @stripesConnect
 export class DataFetcher extends Component {
   static manifest = Object.freeze({
     jobs: {
       type: 'okapi',
-      path: jobsUrl,
+      path: (_q, _p, resources) => getJobSplittingURL(resources, compositeJobsUrl, jobsUrl),
       accumulate: true,
       throwErrors: false,
     },
     logs: {
       type: 'okapi',
-      path: logsUrl,
+      path: (_q, _p, resources) => getJobSplittingURL(resources, compositeLogsUrl, logsUrl),
       accumulate: true,
       throwErrors: false,
     },
+    splitStatus: {
+      type: 'okapi',
+      path: requestConfiguration,
+      shouldRefreshRemote: () => false,
+      throwErrors: false,
+    },
   });
 
   static propTypes = {
@@ -93,29 +115,57 @@ export class DataFetcher extends Component {
           PropTypes.shape({ jobExecutions: PropTypes.arrayOf(jobExecutionPropTypes).isRequired }),
         ).isRequired,
       }),
+      splitStatus: PropTypes.shape({
+        hasLoaded: PropTypes.bool,
+        records: PropTypes.arrayOf(
+          PropTypes.shape({ splitStatus: PropTypes.bool.isRequired }),
+        ).isRequired,
+      }),
     }).isRequired,
     updateInterval: PropTypes.number, // milliseconds
   };
 
   static defaultProps = { updateInterval: DEFAULT_FETCHER_UPDATE_INTERVAL };
 
   state = {
+    statusLoaded: false,
     contextData: { // eslint-disable-line object-curly-newline
       hasLoaded: false,
     },
   };
 
-  async componentDidMount() {
+  componentDidMount() {
+    const { resources:{ splitStatus } } = this.props;
+    const { statusLoaded } = this.state;
     this.mounted = true;
-    await this.fetchResourcesData(true);
-    this.updateResourcesData();
+    this.initialFetchPending = false;
+    if (!statusLoaded && splitStatus?.hasLoaded) {
+      this.setState({ statusLoaded: true }, () => {
+        if (!this.initialFetchPending) this.initialize();
+      });
+    }
+  }
+
+  componentDidUpdate(props, state) {
+    const { resources:{ splitStatus } } = props;
+    const { statusLoaded } = state;
+    if (!statusLoaded && splitStatus?.hasLoaded) {
+      this.setState({ statusLoaded: true }, () => {
+        if (!this.initialFetchPending) this.initialize();
+      });
+    }
   }
 
   componentWillUnmount() {
     this.mounted = false;
     clearTimeout(this.timeoutId);
   }
 
+  initialize = async () => {
+    await this.fetchResourcesData(true);
+    this.updateResourcesData();
+  }
+
   updateResourcesData() {
     const { updateInterval } = this.props;
 
@@ -133,13 +183,14 @@ export class DataFetcher extends Component {
       return;
     }
 
-    const { mutator } = this.props;
+    const { mutator: { jobs, logs } } = this.props;
 
     const fetchResourcesPromises = Object
-      .values(mutator)
+      .values({ jobs, logs })
       .reduce((res, resourceMutator) => res.concat(this.fetchResourceData(resourceMutator)), []);
 
     try {
+      this.initialFetchPending = true;
       await Promise.all(fetchResourcesPromises);
       this.mapResourcesToState();
     } catch (error) {
@@ -163,12 +214,12 @@ export class DataFetcher extends Component {
 
   /** @param  {boolean} [isEmpty] flag to fill contextData with empty data */
   mapResourcesToState(isEmpty) {
-    const { resources } = this.props;
+    const { resources: { jobs, logs } } = this.props;
 
     const contextData = { hasLoaded: true };
 
-    forEach(resources, (resourceValue, resourceName) => {
-      contextData[resourceName] = isEmpty ? [] : get(resourceValue, ['records', 0, 'jobExecutions'], {});
+    forEach({ jobs, logs }, (resourceValue, resourceName) => {
+      contextData[resourceName] = isEmpty ? [] : get(resourceValue, ['records', 0, 'jobExecutions'], []);
     });
 
     this.setState({ contextData });

src/components/DataFetcher/DataFetcher.test.js

@@ -12,6 +12,7 @@ import '../../../test/jest/__mock__';
 import {
   DataFetcher,
   DataFetcherContext,
+  getJobSplittingURL,
 } from '.';
 
 const reset = () => {};
@@ -52,6 +53,12 @@ const resources = buildResources({
       fileName: 'testFileName',
     }],
   }],
+  otherResources: {
+    splitStatus: {
+      hasLoaded: true,
+      records: [{ splitStatus: true }]
+    }
+  }
 });
 
 const TestComponent = () => {
@@ -98,4 +105,21 @@ describe('DataFetcher component', () => {
       await waitFor(() => expect(getByText('error')).toBeDefined());
     });
   });
+
+  describe('getJobSplittingURL', () => {
+    const trueResources = { split_status: { isPending: false, records: [{ splitStatus: true }] } };
+    const falseResources = { split_status: { isPending: false, records: [{ splitStatus: false }] } };
+    const pendingResources = { split_status: { isPending: true, records: [] } };
+    it('given a splitStatus of true, it provides the "trueUrl" parameter', () => {
+      expect(getJobSplittingURL(trueResources, 'trueUrl', 'falseUrl')).toBe('trueUrl');
+    });
+
+    it('given a splitStatus of true, it provides the "falseUrl" parameter', () => {
+      expect(getJobSplittingURL(falseResources, 'trueUrl', 'falseUrl')).toBe('falseUrl');
+    });
+
+    it('given a pending split status, it returns undefined', () => {
+      expect(getJobSplittingURL(pendingResources, 'trueUrl', 'falseUrl')).toBeUndefined();
+    });
+  });
 });

src/components/ImportJobs/ImportJobs.js

@@ -142,7 +142,7 @@ export class ImportJobs extends Component {
 
   onDrop = async acceptedFiles => {
     const { stripes: { okapi } } = this.props;
-    const { updateUploadDefinition } = this.context;
+    const { updateUploadDefinition, uploadConfiguration } = this.context;
 
     const { url: host } = okapi;
 
@@ -175,7 +175,7 @@ export class ImportJobs extends Component {
       return;
     }
 
-    const files = API.mapFilesToUI(acceptedFiles);
+    const files = API.mapFilesToUI(acceptedFiles, uploadConfiguration?.canUseObjectStorage);
 
     try {
       // post file upload definition with all files metadata as

src/components/JobLogsContainer/JobLogsContainer.js

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useContext } from 'react';
 import { useLocation } from 'react-router-dom';
 import { useIntl } from 'react-intl';
 import PropTypes from 'prop-types';
@@ -11,6 +11,7 @@ import {
   stripesShape,
   withStripes,
 } from '@folio/stripes/core';
+import { UploadingJobsContext } from '../UploadingJobsContextProvider/UploadingJobsContext';
 
 import {
   DEFAULT_JOB_LOG_COLUMNS_WIDTHS,
@@ -38,13 +39,21 @@ const JobLogsContainer = props => {
     ...rest
   } = props;
 
-  const { formatMessage } = useIntl();
+  const { formatMessage, formatNumber } = useIntl();
   const location = useLocation();
-
+  const { uploadConfiguration } = useContext(UploadingJobsContext);
   const hasDeletePermission = stripes.hasPerm(permissions.DELETE_LOGS);
 
+  const getVisibleColumns = () => {
+    const baseColumns = [...DEFAULT_JOB_LOG_COLUMNS];
+    if (uploadConfiguration?.canUseObjectStorage) {
+      baseColumns.splice(3, 0, 'jobParts');
+    }
+    return hasDeletePermission ? ['selected', ...baseColumns] : baseColumns;
+  };
+
   const customProperties = {
-    visibleColumns: hasDeletePermission ? ['selected', ...DEFAULT_JOB_LOG_COLUMNS] : DEFAULT_JOB_LOG_COLUMNS,
+    visibleColumns: getVisibleColumns(),
     columnWidths: DEFAULT_JOB_LOG_COLUMNS_WIDTHS,
   };
 
@@ -61,10 +70,12 @@ const JobLogsContainer = props => {
         selectedRecords,
         checkboxDisabled: checkboxesDisabled,
         fieldsConfig,
+        formatNumber,
       }),
       fileName: record => fileNameCellFormatter(record, location),
       status: statusCellFormatter(formatMessage),
       jobProfileName: jobProfileNameCellFormatter,
+      jobParts: record => formatMessage({ id: 'ui-data-import.logViewer.partOfTotal' }, { number: record.jobPartNumber, total: record.totalJobParts }),
     },
   };
 

src/components/JobLogsContainer/JobLogsContainer.test.js

@@ -12,6 +12,8 @@ import '../../../test/jest/__mock__';
 import JobLogsContainer from './JobLogsContainer';
 import { FILE_STATUSES } from '../../utils';
 
+import { UploadingJobsContext } from '../UploadingJobsContextProvider/UploadingJobsContext';
+
 jest.mock('react-router-dom', () => ({
   ...jest.requireActual('react-router-dom'),
   useLocation: () => ({
@@ -20,6 +22,13 @@ jest.mock('react-router-dom', () => ({
   }),
 }));
 
+const splitRecord = {
+  status: FILE_STATUSES.COMMITTED,
+  progress: { current: 0 },
+  jobPartNumber: 2,
+  totalJobParts: 20
+};
+
 const successfulRecord = {
   status: FILE_STATUSES.COMMITTED,
   progress: { current: 0 },
@@ -50,8 +59,16 @@ const checkboxListProp = {
 };
 
 const stripes = buildStripes();
+const defaultUploadContext = {
+  uploadConfiguration: { canUseObjectStorage: { splitStatus: false } }
+};
 
-const renderJobLogsContainer = record => {
+const splittingUploadContext = {
+  uploadConfiguration: { canUseObjectStorage: { splitStatus: true } }
+};
+
+const renderJobLogsContainer = (record, context = defaultUploadContext) => {
+  const { Provider } = UploadingJobsContext;
   const childComponent = listProps => {
     listProps.resultsFormatter.status(record);
     listProps.resultsFormatter.fileName(record);
@@ -60,14 +77,17 @@ const renderJobLogsContainer = record => {
       <div>
         <span>child component</span>
         <span>{listProps.resultsFormatter.status(record)}</span>
+        {record.jobPartNumber && <span>{listProps.resultsFormatter.jobParts(record)}</span>}
       </div>
     );
   };
 
   const component = (
-    <JobLogsContainer checkboxList={checkboxListProp} stripes={stripes}>
-      {({ listProps }) => childComponent(listProps)}
-    </JobLogsContainer>
+    <Provider value={context}>
+      <JobLogsContainer checkboxList={checkboxListProp} stripes={stripes}>
+        {({ listProps }) => childComponent(listProps)}
+      </JobLogsContainer>
+    </Provider>
   );
 
   return renderWithIntl(component, translationsProperties);
@@ -117,4 +137,12 @@ describe('Job Logs container', () => {
       expect(getByText('Stopped by user')).toBeDefined();
     });
   });
+
+  describe('when large file splitting is enabled, display job parts column', () => {
+    it('should render the correct parts current and total for the job', () => {
+      const { getByText } = renderJobLogsContainer(splitRecord, splittingUploadContext);
+      const { jobPartNumber, totalJobParts } = splitRecord;
+      expect(getByText(`${jobPartNumber} of ${totalJobParts}`)).toBeInTheDocument();
+    });
+  });
 });