From cddf5180bb284826aeff4e20486a9cc811a4f6c9 Mon Sep 17 00:00:00 2001 From: Caleb Cox Date: Mon, 11 Nov 2024 09:42:07 -0600 Subject: [PATCH 1/2] Upload TNT exports directly to API and remove file size limit --- .../uploads/upload-tnt-connect-import.page.ts | 89 ------------------- .../uploads/upload-tnt-connect-import.test.ts | 48 ---------- src/components/Tool/TntConnect/TntConnect.tsx | 3 + .../uploads/uploadTntConnect.test.ts | 35 +++----- .../TntConnect/uploads/uploadTntConnect.ts | 40 ++++----- 5 files changed, 35 insertions(+), 180 deletions(-) delete mode 100644 pages/api/uploads/upload-tnt-connect-import.page.ts delete mode 100644 pages/api/uploads/upload-tnt-connect-import.test.ts diff --git a/pages/api/uploads/upload-tnt-connect-import.page.ts b/pages/api/uploads/upload-tnt-connect-import.page.ts deleted file mode 100644 index b2031798d..000000000 --- a/pages/api/uploads/upload-tnt-connect-import.page.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { readFile } from 'fs/promises'; -import { NextApiRequest, NextApiResponse } from 'next'; -import formidable, { IncomingForm } from 'formidable'; -import { getToken } from 'next-auth/jwt'; -import fetch, { File, FormData } from 'node-fetch'; - -export const config = { - api: { - bodyParser: false, - }, -}; - -const parseBody = async ( - req: NextApiRequest, -): Promise<{ fields: formidable.Fields; files: formidable.Files }> => { - return new Promise((resolve, reject) => { - const form = new IncomingForm(); - form.parse(req, (err, fields, files) => { - if (err) { - reject(err); - } else { - resolve({ fields, files }); - } - }); - }); -}; - -const uploadTntConnect = async ( - req: NextApiRequest, - res: NextApiResponse, -): Promise => { - try { - if (req.method !== 'POST') { - res.status(405).send('Method Not Found'); - return; - } - - const jwt = await getToken({ - req, - secret: process.env.JWT_SECRET, - }); - const apiToken = jwt?.apiToken; - if (!apiToken) { - res.status(401).send('Unauthorized'); - return; - } - - const { - fields: { override, tag_list, accountListId }, - files: { file }, - } = await parseBody(req); - if (typeof override !== 'string') { - res.status(400).send('Missing override'); - return; - } - if (!file || Array.isArray(file)) { - res.status(400).send('Missing file'); - return; - } - - const fileUpload = new File( - [await readFile(file.filepath)], - file.originalFilename ?? 'tntConnectUpload', - ); - const form = new FormData(); - form.append('data[type]', 'imports'); - form.append('data[attributes][override]', override); - form.append( - 'data[attributes][tag_list]', - Array.isArray(tag_list) ? tag_list.join(',') : tag_list, - ); - form.append('data[attributes][file]', fileUpload); - const fetchRes = await fetch( - `${process.env.REST_API_URL}account_lists/${accountListId}/imports/tnt`, - { - method: 'POST', - headers: { - authorization: `Bearer ${apiToken}`, - }, - body: form, - }, - ); - res.status(fetchRes.status).json({ success: fetchRes.status === 201 }); - } catch (err) { - res.status(500).json({ success: false, error: err }); - } -}; - -export default uploadTntConnect; diff --git a/pages/api/uploads/upload-tnt-connect-import.test.ts b/pages/api/uploads/upload-tnt-connect-import.test.ts deleted file mode 100644 index fa1c5723a..000000000 --- a/pages/api/uploads/upload-tnt-connect-import.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { getToken } from 'next-auth/jwt'; -import { createMocks } from 'node-mocks-http'; -import uploadTntConnect from './upload-tnt-connect-import.page'; -import 'node-fetch'; - -jest.mock('node-fetch', () => jest.fn()); - -jest.mock('next-auth/jwt', () => ({ getToken: jest.fn() })); -jest.mock('src/lib/apollo/ssrClient', () => jest.fn()); - -const accountListId = 'accountListId'; -const file = new File(['contents1'], 'tnt1.xml', { - type: 'text/xml', -}); -const override = 'false'; -const tag_list = 'tag1'; - -describe('upload-tnt-connect-import', () => { - it('responds with error if unauthorized', async () => { - const { req, res } = createMocks({ - method: 'POST', - body: { - override: override, - file, - tag_list, - accountListId, - }, - }); - - await uploadTntConnect(req, res); - - expect(res._getStatusCode()).toBe(401); - }); - - it('responds with error if not sent with POST', async () => { - (getToken as jest.Mock).mockReturnValue({ - apiToken: 'accessToken', - userID: 'sessionUserID', - }); - const { req, res } = createMocks({ - method: 'GET', - }); - - await uploadTntConnect(req, res); - - expect(res._getStatusCode()).toBe(405); - }); -}); diff --git a/src/components/Tool/TntConnect/TntConnect.tsx b/src/components/Tool/TntConnect/TntConnect.tsx index fd99f66a9..33f92d54b 100644 --- a/src/components/Tool/TntConnect/TntConnect.tsx +++ b/src/components/Tool/TntConnect/TntConnect.tsx @@ -29,6 +29,7 @@ import { LoadingSpinner } from 'src/components/Settings/Organization/LoadingSpin import { ContactTagInput } from 'src/components/Tags/Tags'; import Modal from 'src/components/common/Modal/Modal'; import useGetAppSettings from 'src/hooks/useGetAppSettings'; +import { useRequiredSession } from 'src/hooks/useRequiredSession'; import theme from 'src/theme'; import { ToolsGridContainer } from '../styledComponents'; import { uploadTnt, validateTnt } from './uploads/uploadTntConnect'; @@ -97,6 +98,7 @@ const TntConnect: React.FC = ({ accountListId }: Props) => { const { appName } = useGetAppSettings(); const { t } = useTranslation(); const { enqueueSnackbar } = useSnackbar(); + const { apiToken } = useRequiredSession(); const [showModal, setShowModal] = useState(false); const [loading, setLoading] = useState(false); const [tntFile, setTntFile] = useState<{ @@ -125,6 +127,7 @@ const TntConnect: React.FC = ({ accountListId }: Props) => { if (file) { try { await uploadTnt({ + apiToken, override: attributes.override, selectedTags: attributes.selectedTags, file, diff --git a/src/components/Tool/TntConnect/uploads/uploadTntConnect.test.ts b/src/components/Tool/TntConnect/uploads/uploadTntConnect.test.ts index 5025d5a17..02c432632 100644 --- a/src/components/Tool/TntConnect/uploads/uploadTntConnect.test.ts +++ b/src/components/Tool/TntConnect/uploads/uploadTntConnect.test.ts @@ -1,8 +1,11 @@ +import { session } from '__tests__/fixtures/session'; import { uploadTnt } from './uploadTntConnect'; +const apiToken = session.user.apiToken; + describe('uploadTnt', () => { const fetch = jest.fn().mockResolvedValue({ - json: () => Promise.resolve({ success: true }), + ok: true, }); beforeEach(() => { window.fetch = fetch; @@ -16,16 +19,14 @@ describe('uploadTnt', () => { const textFile = new File(['contents'], 'file.txt', { type: 'text/plain', }); - const largeFile = new File([new ArrayBuffer(2_000_000)], 'large.xml', { - type: 'text/xml', - }); const selectedTags = ['test', 'tag1']; const accountListId = '1234'; - it('uploads the image', () => { + it('uploads the file', () => { return expect( uploadTnt({ + apiToken, override: 'false', selectedTags, accountListId, @@ -38,6 +39,7 @@ describe('uploadTnt', () => { it('rejects files that are not xml files', () => { return expect( uploadTnt({ + apiToken, override: 'false', selectedTags, accountListId, @@ -47,40 +49,29 @@ describe('uploadTnt', () => { ).rejects.toThrow('Cannot upload file: file must be a xml file'); }); - it('rejects files that are too large', () => { - return expect( - uploadTnt({ - override: 'false', - selectedTags, - accountListId, - file: largeFile, - t, - }), - ).rejects.toThrow('Cannot upload file: file size cannot exceed 1MB'); - }); - - it('handles server errors', () => { + it('handles network errors', () => { fetch.mockRejectedValue(new Error('Network error')); return expect( uploadTnt({ + apiToken, override: 'false', selectedTags, accountListId, file, t, }), - ).rejects.toThrow('Cannot upload file: server error'); + ).rejects.toThrow('Cannot upload file: network error'); }); it('handles success being false', () => { - const fetch = jest.fn().mockResolvedValue({ - json: () => Promise.resolve({ success: false }), + fetch.mockResolvedValue({ + ok: false, }); - window.fetch = fetch; return expect( uploadTnt({ + apiToken, override: 'false', selectedTags, accountListId, diff --git a/src/components/Tool/TntConnect/uploads/uploadTntConnect.ts b/src/components/Tool/TntConnect/uploads/uploadTntConnect.ts index aaa7273e3..bc435c4d9 100644 --- a/src/components/Tool/TntConnect/uploads/uploadTntConnect.ts +++ b/src/components/Tool/TntConnect/uploads/uploadTntConnect.ts @@ -13,26 +13,19 @@ export const validateTnt = ({ message: t('Cannot upload file: file must be a xml file'), }; } - // The /api lambda appears to truncate the source body at 2^20 bytes - // Conservatively set the limit at 1MB, which is a little lower than 1MiB because of the - // overhead of encoding multipart/form-data and the other fields in the POST body - if (file.size > 1_000_000) { - return { - success: false, - message: t('Cannot upload file: file size cannot exceed 1MB'), - }; - } return { success: true }; }; export const uploadTnt = async ({ + apiToken, selectedTags, override, file, t, accountListId, }: { + apiToken: string; selectedTags: string[]; override: string; file: File; @@ -45,19 +38,24 @@ export const uploadTnt = async ({ } const form = new FormData(); - form.append('override', override); - form.append('tag_list', selectedTags.join(',')); - form.append('file', file); - form.append('accountListId', accountListId); + form.append('data[type]', 'imports'); + form.append('data[attributes][override]', override); + form.append('data[attributes][tag_list]', selectedTags.join(',')); + form.append('data[attributes][file]', file); - const res = await fetch(`/api/uploads/upload-tnt-connect-import`, { - method: 'POST', - body: form, - }).catch((err) => { - throw new Error(t('Cannot upload file: server error') + err); + const res = await fetch( + `${process.env.REST_API_URL}account_lists/${accountListId}/imports/tnt`, + { + method: 'POST', + headers: { + authorization: `Bearer ${apiToken}`, + }, + body: form, + }, + ).catch(() => { + throw new Error(t('Cannot upload file: network error')); }); - const data: { success: boolean } = await res.json(); - if (!data.success) { - throw new Error(t('Cannot upload file: server not successful') + data); + if (!res.ok) { + throw new Error(t('Cannot upload file: server not successful')); } }; From 31f2251e1669ee33cd56c6d735ae934728baecca Mon Sep 17 00:00:00 2001 From: Caleb Cox Date: Mon, 11 Nov 2024 09:42:53 -0600 Subject: [PATCH 2/2] Clean up TntConnect component * Give variable a more descriptive name * Remove a redundant call to revokeObjectURL --- src/components/Tool/TntConnect/TntConnect.tsx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/components/Tool/TntConnect/TntConnect.tsx b/src/components/Tool/TntConnect/TntConnect.tsx index 33f92d54b..47e16bde3 100644 --- a/src/components/Tool/TntConnect/TntConnect.tsx +++ b/src/components/Tool/TntConnect/TntConnect.tsx @@ -155,9 +155,9 @@ const TntConnect: React.FC = ({ accountListId }: Props) => { const handleFileChange: React.ChangeEventHandler = ( event, ) => { - const f = event.target.files?.[0]; - if (f) { - updateTntFile(f); + const file = event.target.files?.[0]; + if (file) { + updateTntFile(file); } }; @@ -168,6 +168,7 @@ const TntConnect: React.FC = ({ accountListId }: Props) => { } }; }, [tntFile]); + const updateTntFile = (file: File) => { const validationResult = validateTnt({ file, t }); if (!validationResult.success) { @@ -177,10 +178,6 @@ const TntConnect: React.FC = ({ accountListId }: Props) => { return; } - if (tntFile) { - // Release the previous file blob - URL.revokeObjectURL(tntFile.blobUrl); - } setTntFile({ file, blobUrl: URL.createObjectURL(file) }); };