feat(attachments): add attachment mapping file support

specifyweb/frontend/js_src/lib/components/Attachments/index.tsx

@@ -236,7 +236,12 @@ function Attachments({
             <Button.BorderedGray
               onClick={() => navigate('/specify/overlay/attachments/import/')}
             >
-              {commonText.import()}
+              {attachmentsText.importMatchingAttachments()}
+            </Button.BorderedGray>
+            <Button.BorderedGray
+              onClick={() => navigate('/specify/attachments/import//')}
+            >
+              {attachmentsText.importFromMappingFile()}
             </Button.BorderedGray>
           </>
         )}

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Datasets.tsx

@@ -32,6 +32,7 @@ import type {
   AttachmentDataSet,
   AttachmentDataSetPlan,
   FetchedDataSet,
+  PartialUploadableFileSpec,
 } from './types';
 import { useEagerDataSet } from './useEagerDataset';
 
@@ -97,12 +98,21 @@ function ModifyDataset({
   );
 }
 
-const createEmpty = async (name: LocalizedString) =>
+const createEmpty = async (
+  name: LocalizedString,
+  rows: RA<PartialUploadableFileSpec> = []
+) =>
   createEmptyDataSet<AttachmentDataSet>('bulkAttachment', name, {
     uploadplan: { staticPathKey: undefined },
     uploaderstatus: 'main',
+    rows,
   });
 
+export const createEmptyAttachmentDataset = (
+  name: LocalizedString,
+  rows: RA<PartialUploadableFileSpec> = []
+) => createEmpty(name, rows);
+
 export function AttachmentsImportOverlay(): JSX.Element | null {
   const handleClose = React.useContext(OverlayContext);
   const attachmentDataSetsPromise = React.useMemo(fetchAttachmentMappings, []);
@@ -274,7 +284,7 @@ function NewDataSet(): JSX.Element | null {
             id={id('form')}
             onSubmit={async () => {
               loading(
-                createEmpty(pendingName).then(({ id }) =>
+                createEmptyAttachmentDataset(pendingName).then(({ id }) =>
                   navigate(`/specify/attachments/import/${id}`)
                 )
               );

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx

@@ -13,7 +13,7 @@ import { ajax } from '../../utils/ajax';
 import { f } from '../../utils/functools';
 import type { RA, WritableArray } from '../../utils/types';
 import type { IR } from '../../utils/types';
-import { removeKey, sortFunction } from '../../utils/utils';
+import { removeKey, sortFunction, stripFileExtension } from '../../utils/utils';
 import { Container } from '../Atoms';
 import { Button } from '../Atoms/Button';
 import { strictGetTable } from '../DataModel/tables';
@@ -132,19 +132,37 @@ function AttachmentsImport({
     );
 
   const applyFileNames = React.useCallback(
-    (file: UnBoundFile): PartialUploadableFileSpec =>
-      eagerDataSet.uploadplan.staticPathKey === undefined
-        ? { uploadFile: file }
-        : {
-            uploadFile: {
-              ...file,
-              parsedName: resolveFileNames(
-                file.file.name,
-                eagerDataSet.uploadplan.formatQueryResults,
-                eagerDataSet.uploadplan.fieldFormatter
-              ),
-            },
-          },
+    (
+      file: UnBoundFile,
+      preserveParseName = true
+    ): PartialUploadableFileSpec => {
+      if (eagerDataSet.uploadplan.staticPathKey === undefined)
+        return { uploadFile: file };
+      let parsedName = undefined;
+
+      if (file.parsedName !== undefined && preserveParseName)
+        // Maybe we are being too nice. Users should correctly format this?
+        // This wil also correctly handle the case where formatter is no longer valid
+        // because it was changed.
+        parsedName = resolveFileNames(
+          file.parsedName,
+          eagerDataSet.uploadplan.formatQueryResults,
+          eagerDataSet.uploadplan.fieldFormatter
+        );
+      if (parsedName === undefined)
+        parsedName = resolveFileNames(
+          stripFileExtension(file.file.name),
+          eagerDataSet.uploadplan.formatQueryResults,
+          eagerDataSet.uploadplan.fieldFormatter
+        );
+
+      return {
+        uploadFile: {
+          ...file,
+          parsedName,
+        },
+      };
+    },
     [eagerDataSet.uploadplan.staticPathKey]
   );
 
@@ -154,10 +172,15 @@ function AttachmentsImport({
   React.useEffect(() => {
     // Reset all parsed names if matching path is changed
     if (previousKeyRef.current !== eagerDataSet.uploadplan.staticPathKey) {
-      previousKeyRef.current = eagerDataSet.uploadplan.staticPathKey;
       commitFileChange((files) =>
-        files.map(({ uploadFile }) => applyFileNames(uploadFile))
+        files.map(({ uploadFile }) =>
+          // Do not preserve parsed name if previous static key was not undefined.
+          // Otherwise, a numeric formatter can be applied after a text formatter and we will match
+          // records incorrectly
+          applyFileNames(uploadFile, previousKeyRef.current === undefined)
+        )
       );
+      previousKeyRef.current = eagerDataSet.uploadplan.staticPathKey;
     }
   }, [applyFileNames, commitFileChange]);
 
@@ -176,7 +199,11 @@ function AttachmentsImport({
   );
 
   const handleFilesSelected = (files: FileList) => {
-    const filesList = Array.from(files, (file) => applyFileNames({ file }));
+    const filesList: RA<PartialUploadableFileSpec> = Array.from(
+      files,
+      (file) => ({ uploadFile: { file } })
+    );
+
     const oldRows = eagerDataSet.rows;
     const { resolvedFiles, duplicateFiles } = matchSelectedFiles(
       oldRows,
@@ -185,10 +212,13 @@ function AttachmentsImport({
     (resolvedFiles as WritableArray<PartialUploadableFileSpec>).sort(
       sortFunction((file) => file.uploadFile.file.name)
     );
+    const fileNamesApplied = resolvedFiles.map(({ uploadFile }) =>
+      applyFileNames(uploadFile)
+    );
     commitChange((oldState) => ({
       ...oldState,
       uploaderstatus: 'main',
-      rows: resolvedFiles,
+      rows: fileNamesApplied,
     }));
     setDuplicatedFiles(duplicateFiles);
   };

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ImportFromMappingFile.tsx

@@ -0,0 +1,47 @@
+import React from 'react';
+import { localized, RA } from '../../utils/types';
+import { CsvFilePicker } from '../Molecules/CsvFilePicker';
+import { attachmentsText } from '../../localization/attachments';
+import { useNavigate } from 'react-router-dom';
+
+import { PartialUploadableFileSpec } from './types';
+import { createEmptyAttachmentDataset } from './Datasets';
+
+// TODO: Is this needed?
+//const requiredHeaders = ['filename', 'identifier'];
+
+export function ImportFromMappingFile(): JSX.Element {
+  const navigate = useNavigate();
+  return (
+    <>
+      <CsvFilePicker
+        header={attachmentsText.importFromMappingFile()}
+        onFileImport={({ data: rawRows, fileName }) => {
+          // To support mapping file, we just create a new dataset with files. Matching behaviour
+          // within dataset takes care of rest
+          const attachmentRows: RA<PartialUploadableFileSpec> = rawRows.map(
+            ([fileName, parsedName]) => {
+              const trimmedFileName =
+                fileName === undefined ? '' : String(fileName).trim();
+              const trimmedParsedName =
+                parsedName === undefined ? undefined : String(parsedName).trim();
+
+              return {
+                uploadFile: {
+                  file: {
+                    name: trimmedFileName,
+                  },
+                  parsedName: trimmedParsedName,
+                },
+              };
+            }
+          );
+          createEmptyAttachmentDataset(
+            localized(fileName),
+            attachmentRows
+          ).then(({ id }) => navigate(`/specify/attachments/import/${id}`));
+        }}
+      />
+    </>
+  );
+}

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx

@@ -63,7 +63,10 @@ const resolveAttachmentDatasetData = (
             {uploadFile.file.name}
           </div>,
         ],
-        fileSize: formatFileSize(uploadFile.file.size),
+        fileSize:
+          uploadFile.file.size === undefined
+            ? ''
+            : formatFileSize(uploadFile.file.size),
         record: [
           resolvedRecord?.type === 'matched'
             ? resolvedRecord.id

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/ImportFromMappingFile.test.tsx

@@ -0,0 +1,155 @@
+import { waitFor } from '@testing-library/react';
+import React from 'react';
+import fs from 'fs';
+import path from 'path';
+import { jest } from '@jest/globals';
+
+import { mount } from '../../../tests/reactUtils';
+import { LoadingContext } from '../../Core/Contexts';
+import { requireContext } from '../../../tests/helpers';
+import * as Datasets from '../Datasets';
+import { ImportFromMappingFile } from '../ImportFromMappingFile';
+
+requireContext();
+
+const mockNavigate = jest.fn();
+
+jest.mock('react-router-dom', () => ({
+  useNavigate: () => mockNavigate,
+}));
+
+type FileType = 'text/csv' | 'image/jpeg' | 'image/png' | 'text/plain';
+
+const fixturesRoot = path.join(
+  __dirname,
+  '../../../tests/fixtures/attachmentsImport'
+);
+
+function fixtureFile(name: string): File {
+  const fileBuffer = fs.readFileSync(path.join(fixturesRoot, name));
+  const type: FileType = name.endsWith('.png')
+    ? 'image/png'
+    : name.endsWith('.jpg') || name.endsWith('.jpeg')
+    ? 'image/jpeg'
+    : name.endsWith('.csv')
+    ? 'text/csv'
+    : 'text/plain';
+
+  return new File([fileBuffer], name, { type });
+}
+
+describe('ImportFromMappingFile', () => {
+  beforeEach(() => {
+    const portalRoot = document.createElement('div');
+    portalRoot.id = 'portal-root';
+    document.body.appendChild(portalRoot);
+
+    mockNavigate.mockClear();
+    jest
+      .spyOn(Datasets, 'createEmptyAttachmentDataset')
+      .mockResolvedValue({ id: 123 } as any);
+  });
+
+  afterEach(() => {
+    const portalRoot = document.getElementById('portal-root');
+    if (portalRoot !== null) portalRoot.remove();
+    jest.restoreAllMocks();
+  });
+
+  test('creates attachment dataset from a valid CSV mapping file', async () => {
+    const { container, user } = mount(
+      <LoadingContext.Provider value={(promise) => promise as Promise<unknown>}>
+        <ImportFromMappingFile />
+      </LoadingContext.Provider>
+    );
+
+    const fileInput = container.querySelector('input[type=file]');
+    expect(fileInput).toBeTruthy();
+
+    const mappingFile = fixtureFile('mapping-file-valid.csv');
+    await user.upload(fileInput!, mappingFile);
+
+    const importButton = container.getElementsByTagName('button')[0];
+    expect(importButton).toBeTruthy();
+
+    await waitFor(() => expect(importButton).not.toBeDisabled());
+    await user.click(importButton!);
+
+    await waitFor(() => {
+      expect(Datasets.createEmptyAttachmentDataset).toHaveBeenCalledWith(
+        expect.anything(),
+        expect.arrayContaining([
+          expect.objectContaining({
+            uploadFile: expect.objectContaining({
+              file: expect.objectContaining({ name: 'fake-image-1.jpg' }),
+              parsedName: 'ID_001',
+            }),
+          }),
+          expect.objectContaining({
+            uploadFile: expect.objectContaining({
+              file: expect.objectContaining({ name: 'fake-image-2.png' }),
+              parsedName: 'ID_002',
+            }),
+          }),
+        ])
+      );
+    });
+
+    expect(mockNavigate).toHaveBeenCalledWith('/specify/attachments/import/123');
+  });
+
+  test('parses pipe-delimited CSV mapping files with trimmed fields', async () => {
+    const { container, user } = mount(
+      <LoadingContext.Provider value={(promise) => promise as Promise<unknown>}>
+        <ImportFromMappingFile />
+      </LoadingContext.Provider>
+    );
+
+    const fileInput = container.querySelector('input[type=file]');
+    expect(fileInput).toBeTruthy();
+
+    const mappingFile = fixtureFile('mapping-file-pipe-delimited.csv');
+    await user.upload(fileInput!, mappingFile);
+
+    const importButton = container.getElementsByTagName('button')[0];
+    expect(importButton).toBeTruthy();
+
+    await waitFor(() => expect(importButton).not.toBeDisabled());
+    await user.click(importButton!);
+
+    await waitFor(() => {
+      expect(Datasets.createEmptyAttachmentDataset).toHaveBeenCalledWith(
+        expect.anything(),
+        expect.arrayContaining([
+          expect.objectContaining({
+            uploadFile: expect.objectContaining({
+              file: expect.objectContaining({ name: 'fake-image-3.jpg' }),
+              parsedName: 'ID_003',
+            }),
+          }),
+          expect.objectContaining({
+            uploadFile: expect.objectContaining({
+              file: expect.objectContaining({ name: 'fake-image-4.png' }),
+              parsedName: 'ID_004',
+            }),
+          }),
+        ])
+      );
+    });
+
+    expect(mockNavigate).toHaveBeenCalledWith('/specify/attachments/import/123');
+  });
+
+  test('fixture image files are readable and expose correct file metadata', () => {
+    const jpegFile = fixtureFile('fake-image-1.jpg');
+    const pngFile = fixtureFile('fake-image-2.png');
+
+    expect(jpegFile.name).toBe('fake-image-1.jpg');
+    expect(jpegFile.type).toBe('image/jpeg');
+    expect(jpegFile.size).toBeGreaterThan(0);
+
+    expect(pngFile.name).toBe('fake-image-2.png');
+    expect(pngFile.type).toBe('image/png');
+    expect(pngFile.size).toBeGreaterThan(0);
+  });
+});

specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/types.ts

@@ -57,7 +57,8 @@ type UploadableFileSpec = {
  * Forcing keys to be primitive because objects would be
  * ignored during syncing to backend.
  */
-export type BoundFile = Pick<File, 'name' | 'size' | 'type'>;
+export type BoundFile = Pick<File, 'name'> &
+  Partial<Pick<File, 'size' | 'type'>>;
 
 export type UnBoundFile = {
   readonly file: BoundFile | File;