Merge pull request #22 from sonnenkern/develop

Develop

Author: sonnenkern

lib/QuipProcessor.js

@@ -375,7 +375,12 @@ class QuipProcessor {
                 if(file.fileName) {
                     fileName = file.fileName.trim();
                 } else {
-                    fileName = `${file.blobId.trim()}.${Mime.getExtension(blob.type).trim()}`;
+                    const extension = Mime.getExtension(blob.type);
+                    if(extension) {
+                        fileName = `${file.blobId.trim()}.${Mime.getExtension(blob.type).trim()}`;
+                    } else {
+                        fileName = `${file.blobId.trim()}`;
+                    }
                 }
                 fileName = sanitizeFilename(fileName);
 

lib/QuipService.js

@@ -1,11 +1,14 @@
 const fetch = require('node-fetch');
 const LoggerAdapter = require('./common/LoggerAdapter');
 
+const TIMES_LIMIT_503 = 10;
+
 class QuipService {
     constructor(accessToken, apiURL='https://platform.quip.com:443/1') {
         this.accessToken = accessToken;
         this.apiURL = apiURL;
         this.logger = new LoggerAdapter();
+        this.querries503 = new Map();
         this.stats = {
             query_count: 0,
             getThread_count: 0,
@@ -100,15 +103,22 @@ class QuipService {
                 if(res.status === 503) {
                     const waitingInMs = res.headers.get('x-ratelimit-reset')*1000 - new Date().getTime();
                     this.logger.debug(`HTTP 503: for ${url}, waiting in ms: ${waitingInMs}`);
-                    return new Promise(resolve => setTimeout(() => {
-                        resolve(this._apiCallBlob(url, method));
-                    }, waitingInMs));
+                    if(this._check503Query(url)) {
+                        return new Promise(resolve => setTimeout(() => {
+                            resolve(this._apiCallBlob(url, method));
+                        }, waitingInMs));
+                    } else {
+                        this.logger.error(`Couldn't fetch ${url}, tryed to get it ${TIMES_LIMIT_503} times`);
+                        return;
+                    }
                 } else {
                     this.logger.debug(`Couldn't fetch ${url}, received ${res.status}`);
                     return;
                 }
             }
 
+            this.querries503.delete(url);
+
             return res.blob();
         } catch (e) {
             this.logger.error(`Couldn't fetch ${url}, `, e);
@@ -124,9 +134,14 @@ class QuipService {
                 if(res.status === 503) {
                     const waitingInMs = res.headers.get('x-ratelimit-reset')*1000 - new Date().getTime();
                     this.logger.debug(`HTTP 503: for ${url}, waiting in ms: ${waitingInMs}`);
-                    return new Promise(resolve => setTimeout(() => {
-                        resolve(this._apiCall(url, method));
-                    }, waitingInMs));
+                    if(this._check503Query(url)) {
+                        return new Promise(resolve => setTimeout(() => {
+                            resolve(this._apiCall(url, method));
+                        }, waitingInMs));
+                    } else {
+                        this.logger.error(`Couldn't fetch ${url}, tryed to get it ${TIMES_LIMIT_503} times`);
+                        return;
+                    }
                 } else {
                     this.logger.debug(`Couldn't fetch ${url}, received ${res.status}`);
                     return;
@@ -139,6 +154,20 @@ class QuipService {
         }
     }
 
+    _check503Query(url) {
+        let count = this.querries503.get(url);
+        if(!count) {
+            count = 0;
+        }
+
+        this.querries503.set(url, ++count);
+        if(count > TIMES_LIMIT_503) {
+            return false;
+        }
+
+        return true;
+    }
+
     _getOptions(method) {
         return {
             method: method,

lib/__tests__/QuipProcessor.test.js

@@ -698,17 +698,27 @@ describe('methods tests', () => {
             const returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false);
             expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'FILE_NAME_SANITIZED', 'BLOB', '/aaa/blobs');
             expect(returnedValue).toBe('HTML-blobs/FILE_NAME_SANITIZED');
-            //`BLOB_ID.pdf`;
-            //
         });
 
         test('blob processing as file without fileName', async () => {
             fileInfo.fileName = undefined;
             const returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false);
             expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID.pdf_SANITIZED', 'BLOB', '/aaa/blobs');
             expect(returnedValue).toBe('HTML-blobs/BLOB_ID.pdf_SANITIZED');
-            //``;
-            //
+        });
+
+        test('blob processing as file without fileName and without right mime type', async () => {
+            fileInfo.fileName = undefined;
+            //type is not defined
+            delete blob.type;
+            let returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false);
+            expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID_SANITIZED', 'BLOB', '/aaa/blobs');
+            expect(returnedValue).toBe('HTML-blobs/BLOB_ID_SANITIZED');
+            //type is wrong
+            blob.type = "dddfffggg";
+            returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false);
+            expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID_SANITIZED', 'BLOB', '/aaa/blobs');
+            expect(returnedValue).toBe('HTML-blobs/BLOB_ID_SANITIZED');
         });
     });
 

lib/__tests__/QuipService.test.js

@@ -45,6 +45,14 @@ test('_apiCall response with 503 code', async () => {
     expect(quipService.stats.query_count).toBe(2);
 });
 
+test('_apiCall response with 503 code more than 10 times', async () => {
+    quipService.stats.query_count = 0;
+    fetch.mockReturnValue(Promise.resolve(response503));
+    const res = await quipService._apiCall('/someURL');
+    expect(res).toBe(undefined);
+    expect(quipService.logger.error).toHaveBeenCalledWith('Couldn\'t fetch /someURL, tryed to get it 10 times');
+});
+
 test('_apiCall response with 500 code', async () => {
     fetch.mockReturnValue(Promise.resolve(response500));
     await quipService._apiCall('/someURL');
@@ -68,6 +76,14 @@ test('_apiCallBlob response with 503 code', async () => {
     expect(quipService.stats.query_count).toBe(2);
 });
 
+test('_apiCallBlob response with 503 code more than 10 times', async () => {
+    quipService.stats.query_count = 0;
+    fetch.mockReturnValue(Promise.resolve(response503));
+    const res = await quipService._apiCallBlob('/someURL');
+    expect(res).toBe(undefined);
+    expect(quipService.logger.error).toHaveBeenCalledWith('Couldn\'t fetch /someURL, tryed to get it 10 times');
+});
+
 test('_apiCallBlob response with 500 code', async () => {
     fetch.mockReturnValue(Promise.resolve(response500));
     await quipService._apiCallBlob('/someURL');

package.json

@@ -1,6 +1,6 @@
 {
   "name": "quip-export",
-  "version": "2.2.1",
+  "version": "2.2.2",
   "description": "Export all folders and documents from Quip",
   "main": "index.js",
   "jest": {