Skip to content

Commit 8efe1c0

Browse files
EMaslowskiQEMaslowskiQ
authored
(0.9.23) RecordKeeper Introduction + Large File Handling (#637)
* DPO3DPKRT-869/RecordKeeper System and Logger (#632) * DPO3DPKRT-864/Notification Emails (#633) * DPO3DPKRT-863/Slack Notifications (#634) * DPOPKRT-863/Typescript Build Fixes (WebDav Server) (#635) * DPO3DPKRT-879/Large File Upload Connection Drop (#636)
1 parent 8726133 commit 8efe1c0

File tree

37 files changed

+3214
-195
lines changed

37 files changed

+3214
-195
lines changed

‎client/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,6 @@
110110
},
111111
"volta": {
112112
"node": "18.9.0",
113-
"yarn": "1.22.19"
113+
"yarn": "1.22.10"
114114
}
115115
}

‎client/src/graphql/index.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,8 @@ const SAMLRedirectPath: string = '/saml/idp/profile/redirectorpost/sso';
5353
const errorLink = onError(({ graphQLErrors, networkError }) => {
5454
let sentToLogin: boolean = false;
5555

56-
console.log(`[PACKRAT:ERROR] Network: ${JSON.stringify(networkError)}`);
57-
5856
if (graphQLErrors) {
57+
console.log('[PACKRAT:ERROR] apollo processRequest: graphql errors ', graphQLErrors);
5958
graphQLErrors.forEach(({ message, locations, path }) => {
6059
console.log(`[GraphQL error]: Message: ${message}, Location: ${JSON.stringify(locations)}, Path: ${path}`);
6160
if (message.includes(authenticationFailureMessage)) {
@@ -69,6 +68,8 @@ const errorLink = onError(({ graphQLErrors, networkError }) => {
6968
}
7069

7170
if (networkError) {
71+
console.log('[PACKRAT:ERROR] apollo processRequest: network errors ', networkError);
72+
7273
if (!sentToLogin) {
7374
let redirectToLogin: boolean = false;
7475

@@ -178,6 +179,8 @@ interface IApolloUploader {
178179
async function apolloUploader(options: IApolloUploader): Promise<any> {
179180
const { mutation, variables, useUpload, refetchQueries, onProgress, onCancel } = options;
180181

182+
console.log('[PACKRAT] apolloUploader',options);
183+
181184
// console.log('apolloUploader apolloClient.mutate');
182185
return await apolloClient.mutate({
183186
mutation,

‎client/src/pages/Admin/components/AdminToolsView.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ const SelectScenesTable = <T extends DBReference>({ onUpdateSelection, data, col
438438
{ (data && Array.isArray(data)) ?
439439
stableSort(data, getComparator(order, orderBy))
440440
.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage)
441-
.map((row, index) => {
441+
.map((row: T, index: number) => {
442442
const isItemSelected = isSelected(row);
443443
const labelId = `table-checkbox-${index}`;
444444

‎client/src/store/upload.ts‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
214214
const attachmentFile = pendingAttachments.get(idSystemObject) as IngestionFile;
215215
const file = isUpdate ? updateFile : isAttachment ? attachmentFile : getFile(id, pending);
216216

217+
console.log(`[PACKRAT] startUpload (${updateFile?.file?.name ?? 'na'}, ${updateFile?.file?.size ?? '-1'})`);
218+
217219
if (file) {
218220
if (isUpdate) {
219221
file.progress = 0;
@@ -276,6 +278,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
276278
const { pending } = get();
277279
const file = getFile(id, pending);
278280

281+
console.log(`[PACKRAT] cancelUpload (${file?.name ?? 'undefined'})`);
282+
279283
if (file) {
280284
if (file.status === FileUploadStatus.UPLOADING) {
281285
const { cancel } = file;
@@ -343,22 +347,32 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
343347

344348
//This is the uploading for the Processed Files
345349
startUploadTransfer: async (ingestionFile: IngestionFile, references?: UploadReferences) => {
350+
351+
console.log(`[PACKRAT] startUploadTransfer (${ingestionFile.file.name})`);
352+
346353
const { pending } = get();
347354
const { id, file, type } = ingestionFile;
348355
try {
349356
const onProgress = (event: ProgressEvent) => {
350357
// registered with Apollo xhr.upload.onprogress callback and triggers
351358
// PROGRESS event at every 10 steps so the UX can properly update.
352359
const { loaded, total } = event;
360+
361+
if (total === 0) {
362+
console.warn('[PACKRAT] ProgressEvent total is zero. Unable to calculate progress.');
363+
return;
364+
}
365+
353366
const progress = Math.floor((loaded / total) * 100);
354-
const updateProgress = (progress % 10) === 0;
367+
const updateProgress = (progress % 5) === 0;
355368

356369
if (updateProgress) {
357370
const progressEvent: UploadProgressEvent = {
358371
id,
359372
progress
360373
};
361374
// console.log(`[PACKRAT] startUploadTransfer.onProgress fire event (event: ${progressEvent} | pending: ${JSON.stringify(pending)})`);
375+
console.log('[PACKRAT] upload onProgress',event);
362376
UploadEvents.dispatch(UploadEventType.PROGRESS, progressEvent);
363377
}
364378
};
@@ -372,10 +386,11 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
372386
// console.log(`[PACKRAT] startUploadTransfer.onCancel fire event (event: ${cancelEvent} | pending: ${JSON.stringify(pending)})`);
373387
UploadEvents.dispatch(UploadEventType.SET_CANCELLED, cancelEvent);
374388
};
375-
const onFailed = () => {
389+
const onFailed = (error: ErrorEvent) => {
376390
// registered with Apollo xhr.upload.onerror and triggers a failed event
377391
const failedEvent: UploadFailedEvent = {
378392
id,
393+
message: error.message || 'Unknown Error'
379394
};
380395

381396
// console.error(`[PACKRAT] startUploadTransfer.onFailed fire event (event: ${error})`);
@@ -392,6 +407,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
392407
uploadAssetInputs.type = getVocabularyId(eVocabularyID.eAssetAssetTypeAttachment) ?? 0;
393408
}
394409

410+
console.log(`[PACKRAT] uploading file '${file?.name ?? 'na'}' of size: ${file?.size ?? '-1'}`);
411+
395412
const { data } = await apolloUploader({
396413
mutation: UploadAssetDocument,
397414
variables: uploadAssetInputs,
@@ -414,14 +431,14 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
414431
toast.success(`Upload finished for ${file.name}`);
415432
} else if (status === UploadStatus.Failed) {
416433
console.log(`[PACKRAT] startUploadTransfer upload failed (id: ${id} | file: ${(file)?file['path']:'na'} | error: ${error})`);
417-
const failedEvent: UploadFailedEvent = { id };
434+
const failedEvent: UploadFailedEvent = { id, message: error || 'Unknown error' };
418435
UploadEvents.dispatch(UploadEventType.FAILED, failedEvent);
419436

420437
const errorMessage = error || `Upload failed for ${file.name}`;
421438
toast.error(errorMessage);
422439
} else if (status === UploadStatus.Noauth) {
423440
console.log(`[PACKRAT:ERROR] startUploadTransfer upload failed ${id}, ${JSON.stringify(file)}, user not authenticated`);
424-
const failedEvent: UploadFailedEvent = { id };
441+
const failedEvent: UploadFailedEvent = { id, message: error || 'Unknown error' };
425442
UploadEvents.dispatch(UploadEventType.FAILED, failedEvent);
426443

427444
global.alert('The Packrat user is no longer authenticated. Please login.');
@@ -436,7 +453,7 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
436453
if (file) {
437454
if (file.status !== FileUploadStatus.CANCELLED) {
438455
console.log(`[PACKRAT:ERROR] startUploadTransfer upload failed ${id}, ${JSON.stringify(file)}, exception ${message}`);
439-
const failedEvent: UploadFailedEvent = { id };
456+
const failedEvent: UploadFailedEvent = { id, message: 'cancelled' };
440457
UploadEvents.dispatch(UploadEventType.FAILED, failedEvent);
441458
}
442459
}
@@ -515,6 +532,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
515532
// });
516533
// console.log(`[PACKRAT] onProgressEvent (options: ${JSON.stringify(options)} | pending: ${JSON.stringify(statusPending)})`);
517534

535+
console.log(`[PACKRAT] onProgress (${progress})`);
536+
518537
// handles updates and attachments if a SystemObject id is provided
519538
if (options?.idSystemObject) {
520539
const isAttachment = options.references.idSOAttachment;
@@ -572,6 +591,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
572591
// });
573592
// console.log(`[PACKRAT] onCancelledEvent (pending: ${JSON.stringify(statusPending)})`);
574593

594+
console.log(`[PACKRAT] onCancel (id: ${id})`);
595+
575596
// handles updates and attachments
576597
if (options?.idSystemObject) {
577598
const isAttachment = options.references.idSOAttachment;
@@ -620,6 +641,7 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
620641
// };
621642
// });
622643
// console.log(`[PACKRAT] onFailedEvent (pending: ${JSON.stringify(statusPending)})`);
644+
console.log(`[PACKRAT] onFailed (id: ${id})`);
623645

624646
// handles updates and attachments
625647
if (options?.idSystemObject) {
@@ -699,6 +721,8 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
699721
}
700722
},
701723
reset: (): void => {
724+
console.log('[PACKRAT] reset');
725+
702726
const { completed, /*pending*/ } = get();
703727
const unselectFiles = (file: IngestionFile): IngestionFile => ({
704728
...file,
@@ -721,6 +745,7 @@ export const useUploadStore = create<UploadStore>((set: SetState<UploadStore>, g
721745
set({ completed: [ ...updatedCompleted ], loading: false });
722746
},
723747
resetSpecialPending: (uploadType: eIngestionMode) => {
748+
console.log('[PACKRAT] reset special');
724749
if (uploadType === eIngestionMode.eAttach) set({ pendingAttachments: new Map() });
725750
if (uploadType === eIngestionMode.eUpdate) set({ pendingUpdates: new Map() });
726751
}

‎client/src/utils/events.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@ export type UploadSetCancelEvent = {
1919

2020
export type UploadFailedEvent = {
2121
id: FileId;
22+
message: string;
2223
};
2324

2425
export type UploadCompleteEvent = {
2526
id: FileId;
2627
};
2728

2829

29-
export type UploadEventData = UploadProgressEvent | UploadFailedEvent;
30+
export type UploadEventData = UploadProgressEvent | UploadFailedEvent | UploadCompleteEvent;
3031

3132
class UploadEvents {
3233
static subscribe(event: UploadEventType, listener: EventListenerOrEventListenerObject): void {

‎common/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,6 @@
3939
},
4040
"volta": {
4141
"node": "18.9.0",
42-
"yarn": "1.22.19"
42+
"yarn": "1.22.10"
4343
}
4444
}

‎conf/docker/docker-compose.dev.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ services:
1313
- ../../conf/nginx/nginx-dev.conf:/etc/nginx/nginx.conf
1414
ports:
1515
- $PACKRAT_PROXY_PORT:80
16+
environment:
17+
- NGINX_PROXY_READ_TIMEOUT=86400
18+
- NGINX_PROXY_CONNECT_TIMEOUT=86400
19+
- NGINX_PROXY_SEND_TIMEOUT=86400
1620

1721
packrat-client:
1822
container_name: packrat-client

‎conf/docker/docker-compose.prod.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ services:
1111
target: proxy
1212
ports:
1313
- $PACKRAT_PROXY_PORT:80
14+
environment:
15+
- NGINX_PROXY_READ_TIMEOUT=86400
16+
- NGINX_PROXY_CONNECT_TIMEOUT=86400
17+
- NGINX_PROXY_SEND_TIMEOUT=86400
1418

1519
packrat-client:
1620
container_name: packrat-client

‎conf/nginx/conf.d/common-locations-dev‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,20 @@ location /server {
44
proxy_set_header Host $host//server;
55
proxy_set_header Upgrade $http_upgrade;
66
proxy_set_header Connection "upgrade";
7-
proxy_read_timeout 3600;
8-
proxy_connect_timeout 3600;
9-
proxy_send_timeout 3600;
7+
8+
# Increase timeouts for large file uploads
9+
proxy_read_timeout 86400; # 24 hours
10+
proxy_send_timeout 86400; # 24 hours
11+
proxy_connect_timeout 86400; # 24 hours
12+
keepalive_timeout 86400; # 24 hours
13+
14+
# Disable request buffering for large file uploads
15+
proxy_request_buffering off;
16+
proxy_buffering off;
17+
18+
# Set headers for persistent connections
19+
proxy_set_header Connection "Keep-Alive";
20+
proxy_set_header Keep-Alive "timeout=600";
1021
}
1122

1223
location / {

‎conf/nginx/conf.d/common-locations-prod‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,20 @@ location /server {
44
proxy_set_header Host $host//server;
55
proxy_set_header Upgrade $http_upgrade;
66
proxy_set_header Connection "upgrade";
7-
proxy_read_timeout 3600;
8-
proxy_connect_timeout 3600;
9-
proxy_send_timeout 3600;
7+
8+
# Increase timeouts for large file uploads
9+
proxy_read_timeout 86400; # 24 hours
10+
proxy_send_timeout 86400; # 24 hours
11+
proxy_connect_timeout 86400; # 24 hours
12+
keepalive_timeout 86400; # 24 hours
13+
14+
# Disable request buffering for large file uploads
15+
proxy_request_buffering off;
16+
proxy_buffering off;
17+
18+
# Set headers for persistent connections
19+
proxy_set_header Connection "Keep-Alive";
20+
proxy_set_header Keep-Alive "timeout=600";
1021
}
1122

1223
location / {

0 commit comments

Comments
 (0)