[CORE-132] Autogenerate dataTable from Data Uploader (#5174)

src/libs/ajax/workspaces/Workspaces.ts

@@ -478,6 +478,12 @@ export const Workspaces = (signal?: AbortSignal) => ({
           _.mergeAll([authOpts(), { signal }])
         ).then((r) => r.blob()),
 
+      autoGenerateTsv: (entityType: string, prefix: string): Promise<Blob> =>
+        fetchOrchestration(
+          `api/workspaces/${namespace}/${name}/entities/${entityType}/paired-tsv`,
+          _.mergeAll([authOpts(), jsonBody({ prefix }), { signal, method: 'POST' }])
+        ).then((r: { blob: () => any }) => r.blob()),
+
       copyEntities: async (
         destNamespace: string,
         destName: string,

src/libs/events.ts

@@ -94,6 +94,7 @@ const eventsList = {
   uploaderUploadMetadata: 'uploader:metadata:upload',
   uploaderCreateTable: 'uploader:table:create',
   uploaderUpdateTable: 'uploader:table:update',
+  uploaderAutoGenerateTable: 'uploader:table:autoGenerate',
   user: {
     authToken: {
       load: {

src/libs/feature-previews-config.ts

@@ -7,6 +7,7 @@ export const FIRECLOUD_UI_MIGRATION = 'firecloudUiMigration';
 export const COHORT_BUILDER_CARD = 'cohortBuilderCard';
 export const GCP_BUCKET_LIFECYCLE_RULES = 'gcpBucketLifecycleRules';
 export const SPEND_REPORTING = 'spendReporting';
+export const AUTO_GENERATE_DATA_TABLES = 'autoGenerateDataTables';
 
 // If the groups option is defined for a FeaturePreview, it must contain at least one group.
 type GroupsList = readonly [string, ...string[]];
@@ -126,6 +127,16 @@ const featurePreviewsConfig: readonly FeaturePreview[] = [
     )}`,
     lastUpdated: '11/19/2024',
   },
+  {
+    id: AUTO_GENERATE_DATA_TABLES,
+    title: 'Autogenerate data table for single and paired end sequencing',
+    description:
+      'Enabling this feature will show a new option in the data uploader to autogenerate a data table instead of uploading your own TSV. This feature will attempt to automatch known file patterns for single and paired end sequencing and autogenerate a data table linking to those files.',
+    feedbackUrl: `mailto:[email protected]?subject=${encodeURIComponent(
+      'Feedback on Autogenerate data table for single and paired end sequencing'
+    )}`,
+    lastUpdated: '11/26/2024',
+  },
 ];
 
 export default featurePreviewsConfig;

src/workspace-data/upload-data/UploadData.js

@@ -2,7 +2,7 @@ import { readFileAsText } from '@terra-ui-packages/core-utils';
 import _ from 'lodash/fp';
 import { Fragment, useEffect, useMemo, useRef, useState } from 'react';
 import { code, div, h, h2, h3, li, p, span, strong, ul } from 'react-hyperscript-helpers';
-import { ButtonPrimary, Link, topSpinnerOverlay, transparentSpinnerOverlay, VirtualizedSelect } from 'src/components/common';
+import { ButtonOutline, ButtonPrimary, Link, topSpinnerOverlay, transparentSpinnerOverlay, VirtualizedSelect } from 'src/components/common';
 import FileBrowser from 'src/components/data/FileBrowser';
 import Dropzone from 'src/components/Dropzone';
 import FloatingActionButton from 'src/components/FloatingActionButton';
@@ -17,6 +17,8 @@ import { Workspaces } from 'src/libs/ajax/workspaces/Workspaces';
 import colors from 'src/libs/colors';
 import { reportError, withErrorReporting } from 'src/libs/error';
 import Events from 'src/libs/events';
+import { isFeaturePreviewEnabled } from 'src/libs/feature-previews';
+import { AUTO_GENERATE_DATA_TABLES } from 'src/libs/feature-previews-config';
 import * as Nav from 'src/libs/nav';
 import { forwardRefWithName, useCancellation, useOnMount } from 'src/libs/react-utils';
 import * as StateHistory from 'src/libs/state-history';
@@ -564,10 +566,12 @@ const MetadataUploadPanel = ({
   const basePrefix = `${rootPrefix}${collection}/`;
   const [filesLoading, setFilesLoading] = useState(false);
   const [uploading, setUploading] = useState(false);
+  const [autoGenerating, setAutoGenerating] = useState(false);
 
   const [metadataFile, setMetadataFile] = useState(null);
   const [metadataTable, setMetadataTable] = useState(null);
   const [filenames, setFilenames] = useState({});
+  const [autoGeneratedFile, setAutoGeneratedFile] = useState(null);
 
   const setErrors = (errors) => {
     setMetadataTable({ errors });
@@ -762,10 +766,54 @@ const MetadataUploadPanel = ({
     }
   });
 
+  const handleAutoGeneratedFile = (autoGeneratedFile, eventName, eventStatus) => {
+    const isDone = eventStatus !== 'generated';
+    if (autoGeneratedFile) {
+      void Metrics().captureEvent(eventName, {
+        ...autoGeneratedFile,
+        status: eventStatus,
+      });
+      isDone && setAutoGeneratedFile(null);
+    }
+  };
+
+  const doAutoGenerate = _.flow(Utils.withBusyState(setAutoGenerating))(async () => {
+    try {
+      const workspace = Workspaces().workspace(namespace, name);
+
+      setAutoGeneratedFile({
+        workspaceNamespace: namespace,
+        workspaceName: name,
+        entityType: collection,
+        prefix: basePrefix,
+      });
+
+      const autoGeneratedTsv = await workspace.autoGenerateTsv(collection, basePrefix);
+      setMetadataFile(autoGeneratedTsv);
+
+      handleAutoGeneratedFile(autoGeneratedFile, Events.uploaderAutoGenerateTable, 'generated');
+    } catch (error) {
+      handleAutoGeneratedFile(autoGeneratedFile, Events.uploaderAutoGenerateTable, 'failed');
+      await reportError('Failed to autogenerate entity metadata', error);
+    }
+  });
+
   // Render
 
+  const renderAutoGenerateSection = () => {
+    if (!isFeaturePreviewEnabled(AUTO_GENERATE_DATA_TABLES)) return null;
+
+    return div({ style: { display: 'flex', flexDirection: 'column', alignItems: 'center' } }, [
+      h(ButtonOutline, { style: { ...styles.heading, flex: 0 }, onClick: doAutoGenerate }, [
+        'Autogenerate table for single or paired end sequencing (BETA)',
+      ]),
+      h2({ style: { ...styles.heading, flex: 0, margin: '0.25rem' } }, [span({ ref: header, tabIndex: -1 }, ['OR'])]),
+    ]);
+  };
+
   return div({ style: { height: '100%', display: 'flex', flexFlow: 'column nowrap' } }, [
-    h2({ style: { ...styles.heading, flex: 0 } }, [
+    renderAutoGenerateSection(),
+    h2({ style: { ...styles.heading, flex: 0, margin: '0.25rem' } }, [
       icon('listAlt', { size: 20, style: { marginRight: '1ch' } }),
       span({ ref: header, tabIndex: -1 }, ['Upload Your Metadata Files']),
     ]),
@@ -859,14 +907,18 @@ const MetadataUploadPanel = ({
         metadataTable,
         onConfirm: ({ metadata }) => {
           doUpload(metadata);
+          // Track if autogenerated table is accepted
+          handleAutoGeneratedFile(autoGeneratedFile, Events.uploaderAutoGenerateTable, 'accepted');
         },
         onCancel: () => {
           setMetadataFile(null);
           setMetadataTable(null);
+          // Track if autogenerated table is cancelled
+          handleAutoGeneratedFile(autoGeneratedFile, Events.uploaderAutoGenerateTable, 'cancelled');
         },
         onRename: renameTable,
       }),
-    (filesLoading || uploading) && topSpinnerOverlay,
+    (filesLoading || uploading || autoGenerating) && topSpinnerOverlay,
   ]);
 };
 

src/workspace-data/upload-data/UploadData.test.tsx

@@ -0,0 +1,137 @@
+import { expect } from '@storybook/test';
+import { act, fireEvent, screen } from '@testing-library/react';
+import React from 'react';
+import { Workspaces, WorkspacesAjaxContract } from 'src/libs/ajax/workspaces/Workspaces';
+import { useRoute } from 'src/libs/nav';
+import { asMockedFn, partial, renderWithAppContexts } from 'src/testing/test-utils';
+
+import { UploadData } from './UploadData';
+
+// Mock dependencies
+jest.mock('src/libs/ajax/GoogleStorage', () => ({
+  GoogleStorage: jest.fn().mockReturnValue({
+    list: jest.fn().mockResolvedValue({ prefixes: [] }),
+    listAll: jest.fn().mockResolvedValue({ items: [] }),
+  }),
+}));
+
+jest.mock('src/libs/ajax/workspaces/Workspaces', () => ({
+  Workspaces: jest.fn().mockReturnValue({
+    workspace: jest.fn().mockReturnValue({
+      importFlexibleEntitiesFileSynchronous: jest.fn().mockResolvedValue({}),
+      autoGenerateTsv: jest.fn().mockResolvedValue(new File([], 'autoGenerated.tsv')),
+    }),
+  }),
+}));
+
+asMockedFn(Workspaces).mockReturnValue(
+  partial<WorkspacesAjaxContract>({
+    getTags: jest.fn().mockResolvedValue([]),
+  })
+);
+
+jest.mock('src/libs/ajax/Metrics', () => ({
+  Metrics: jest.fn().mockReturnValue({
+    captureEvent: jest.fn(),
+  }),
+}));
+
+type NavExports = typeof import('src/libs/nav');
+jest.mock(
+  'src/libs/nav',
+  (): NavExports => ({
+    ...jest.requireActual<NavExports>('src/libs/nav'),
+    getLink: jest.fn(() => '/'),
+    goToPath: jest.fn(),
+    useRoute: jest.fn().mockReturnValue({ query: {} }),
+    updateSearch: jest.fn(),
+  })
+);
+
+jest.mock('src/libs/state-history', () => ({
+  get: jest.fn().mockReturnValue({}),
+  update: jest.fn(),
+}));
+
+jest.mock('src/workspaces/common/state/useWorkspaces', () => ({
+  useWorkspaces: jest.fn().mockReturnValue({
+    workspaces: [
+      {
+        workspace: {
+          workspaceId: 'mockWorkspaceId',
+          namespace: 'mockNamespace',
+          name: 'mockWorkspaceName',
+          attributes: {},
+          createdBy: '[email protected]',
+          lastModified: '2023-01-01T00:00:00Z',
+        },
+        accessLevel: 'OWNER',
+      },
+    ],
+    refresh: jest.fn(),
+    loading: false,
+  }),
+}));
+
+jest.mock('src/libs/feature-previews', () => ({
+  isFeaturePreviewEnabled: jest.fn(),
+}));
+
+describe('UploadData Component', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  const selectOptionAndGoToNext = async (option: any) => {
+    (useRoute as jest.Mock).mockReturnValue({
+      query: { ...option },
+    });
+
+    await act(async () => {
+      renderWithAppContexts(<UploadData />);
+    });
+
+    fireEvent.click(screen.getByText('Next >'));
+  };
+
+  it('Renders data uploader', () => {
+    renderWithAppContexts(<UploadData />);
+    expect(screen.getByRole('main')).toBeInTheDocument();
+    expect(screen.getByText('Data Uploader')).toBeInTheDocument();
+  });
+
+  it('Displays option to select a workspace', () => {
+    renderWithAppContexts(<UploadData />);
+    expect(screen.getByText('Select a Workspace')).toBeInTheDocument();
+  });
+
+  it('Allows selection of a workspace and move to the next step', async () => {
+    // Select workspace
+    await selectOptionAndGoToNext({ workspace: 'mockWorkspaceId' });
+    await expect(screen.getByText('Select a collection')).toBeInTheDocument();
+  });
+
+  it('Allows selection of a collection and move to the next step', async () => {
+    // Select workspace & collection
+    await selectOptionAndGoToNext({ workspace: 'mockWorkspaceId', collection: 'collection1' });
+    await expect(screen.getByText('Upload Your Data Files')).toBeInTheDocument();
+  });
+
+  it('Can manually upload a metadata TSV file', async () => {
+    // Select workspace & collection
+    await selectOptionAndGoToNext({ workspace: 'mockWorkspaceId', collection: 'collection1' });
+
+    // Upload metadata file
+    const file = new File(['entity:collection1_id\tname\n1\tTest'], 'metadata.tsv', {
+      type: 'text/tab-separated-values',
+    });
+    const input = screen.getByTestId('dropzone-upload') as HTMLInputElement;
+    Object.defineProperty(input, 'files', { value: [file], writable: false });
+
+    await act(async () => {
+      fireEvent.change(input);
+    });
+
+    await expect(screen.getByRole('table')).toBeInTheDocument();
+  });
+});