diff --git a/.gitignore b/.gitignore index 4b86977..06ab92a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .DS_Store dist node_modules +staging diff --git a/package.json b/package.json index a748d55..41c758e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bu-protected-s3-object-lambda", - "version": "1.0.7", + "version": "1.0.8", "description": "Delivers asset from S3 through an object lambda with access restrictions and image resizing", "type": "module", "scripts": { diff --git a/src/getOrCreateObject/getOrCreateObject.js b/src/getOrCreateObject/getOrCreateObject.js index 636c254..3313790 100644 --- a/src/getOrCreateObject/getOrCreateObject.js +++ b/src/getOrCreateObject/getOrCreateObject.js @@ -26,7 +26,7 @@ async function getOrCreateObject(userRequest, domain) { const { url } = userRequest; const { pathname, searchParams } = new URL(url); // Get the size match from the pathname. - const sizeMatch = pathname.match(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif)$/); + const sizeMatch = pathname.match(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif|webp)$/); // Decode the pathname for unicode characters. const decodedPathname = decodeURIComponent(pathname); @@ -67,7 +67,7 @@ async function getOrCreateObject(userRequest, domain) { // if the image is not found, and there is a size match, then resize the image and save it to S3. if (response.Code === 'NoSuchKey' && sizeMatch) { // Reconstruct what the original image s3 key would be, by removing the image size from the URL - const originalPath = decodedPathname.replace(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif)$/, '.$3'); + const originalPath = decodedPathname.replace(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif|webp)$/, '.$3'); const originalKey = `${ORIGINAL_PATH_ROOT}/${domain}${originalPath}`; const originalResponse = await tryGetObject(userRequest, originalKey); diff --git a/src/getOrCreateObject/getOrCreateObject.test.js b/src/getOrCreateObject/getOrCreateObject.test.js index 4f3f152..180225c 100644 --- a/src/getOrCreateObject/getOrCreateObject.test.js +++ b/src/getOrCreateObject/getOrCreateObject.test.js @@ -15,6 +15,10 @@ process.env.ORIGINAL_BUCKET = 'test-bucket'; // so that the sharp library has something to resize. const singlePixelJpgReadable = Readable.from(Buffer.from('/9j/4AAQSkZJRgABAQEAAAAAAAD/2wBDAAoHBwkHBgoJCAkLCwoMDxkQDw4ODx4WFxIZJCAmJSMgIyIoLTkwKCo2KyIjMkQyNjs9QEBAJjBGS0U+Sjk/QD3/2wBDAQsLCw8NDx0QEB09KSMpPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT3/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9/KKKKAP/2Q==', 'base64')); +// Create a 1 pixel webp image as a readable stream, +// so that the sharp library has something to resize. +const singlePixelWebpReadable = Readable.from(Buffer.from('UklGRjwAAABXRUJQVlA4IDAAAADQAQCdASoBAAEAAUAmJaACdLoB+AADsAD+8ut//NgVzXPv9//S4P0uD9Lg/9KQAAA=', 'base64')); + // Mock the ddb client. const ddbMock = mockClient(DynamoDBDocumentClient); ddbMock.on(GetCommand).resolves({ @@ -54,6 +58,14 @@ s3Mock.on(GetObjectCommand, { Body: singlePixelJpgReadable, }); +// Define an existing webp object in the bucket, that returns a valid and rescalable image. +s3Mock.on(GetObjectCommand, { + Bucket: 'test-bucket', + Key: 'original_media/www.bu.edu/somesite/files/01/exists.webp', +}).resolves({ + Body: singlePixelWebpReadable, +}); + describe('getOrCreateObject', () => { it('should return an object if it exists', async () => { const result = await getOrCreateObject( @@ -99,6 +111,17 @@ describe('getOrCreateObject', () => { expect(result.Code).toEqual('NoSuchKey'); }); + it('should find the unscaled webp original for the request, scale it, and return an object', async () => { + const result = await getOrCreateObject( + { + url: 'https://example-1111.s3-object-lambda.us-east-1.amazonaws.com/somesite/files/01/exists-300x200.webp', + headers: { }, + }, + 'www.bu.edu', + ); + expect(result.Body).toBeDefined(); + }); + it('should return a pre-scaled original object if it exists', async () => { const result = await getOrCreateObject( { diff --git a/src/getOrCreateObject/resizeAndSave.js b/src/getOrCreateObject/resizeAndSave.js index f73a880..4bb6fc3 100644 --- a/src/getOrCreateObject/resizeAndSave.js +++ b/src/getOrCreateObject/resizeAndSave.js @@ -13,18 +13,12 @@ const bucketName = process.env.ORIGINAL_BUCKET; const s3 = new S3(); -function getOriginalS3Key(url) { - // Reconstruct what the original image s3 key would be, by removing the image size from the URL. - const originalUrl = url.replace(/-(\d+)x(\d+)\.(jpg|png)$/, '.$3'); - const parsedUrl = new URL(originalUrl); - const { pathname } = parsedUrl; - // The s3 key is the pathname without the leading slash. - const s3Key = pathname.replace(/^\//, ''); - - return s3Key; -} +// Sharp won't decode an image with more pixels than its limit, 268402689 pixels by default. +// Animated images count every frame toward that limit, so a long animation can exceed it. +const PIXEL_LIMIT_MESSAGE = 'exceeds pixel limit'; // Resize and save the image to S3, then return the resized image data. +// If the image is too large for sharp to resize, return the original image data instead. async function resizeAndSave(data, originalPath, sizeMatch, crop) { // Get the width and height from the sizeMatch as integers. const width = parseInt(sizeMatch[1], 10); @@ -41,19 +35,33 @@ async function resizeAndSave(data, originalPath, sizeMatch, crop) { // This used to not be necessary but changed with the v3 S3 SDK. const imageBuffer = await streamToString(data.Body); - // Resize the image data with sharp. - const resized = await sharp(imageBuffer).resize({ - width, - height, - ...options, - }).withMetadata(); + // Resize the image data with sharp, and get the resized image data as a buffer. + let resized; + let resizedBuffer; + try { + resized = await sharp(imageBuffer, { animated: true }).resize({ + width, + height, + ...options, + }).withMetadata(); + + resizedBuffer = await resized.toBuffer(); + } catch (error) { + // Anything other than the pixel limit is unexpected, so let it propagate. + if (!error?.message?.includes(PIXEL_LIMIT_MESSAGE)) { + throw error; + } + + // This image can't be resized, so serve the original unscaled for this request. + // Nothing is saved to S3, because there is no resized image to save. + console.log('Resize skipped, the image exceeds the sharp pixel limit: ', originalPath); + + return imageBuffer; + } // Strip file extension from the original s3Key. const pathWithoutExtension = originalPath.replace(/\.[^/.]+$/, ''); - // Get the resized image data as a buffer. - const resizedBuffer = await resized.toBuffer(); - // Encode the original path, so that it can be used in the metadata. const encodedPath = encodeURI(originalPath); @@ -71,4 +79,4 @@ async function resizeAndSave(data, originalPath, sizeMatch, crop) { return resized; } -export { getOriginalS3Key, resizeAndSave }; +export { resizeAndSave }; diff --git a/src/getOrCreateObject/resizeAndSave.test.js b/src/getOrCreateObject/resizeAndSave.test.js new file mode 100644 index 0000000..bf6d283 --- /dev/null +++ b/src/getOrCreateObject/resizeAndSave.test.js @@ -0,0 +1,160 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import { + describe, it, expect, vi, beforeEach, +} from 'vitest'; +import { mockClient } from 'aws-sdk-client-mock'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Readable } from 'stream'; + +// Import the function to test. +// Vitest hoists the vi.hoisted() and vi.mock() calls below above this import, +// so the bucket name is set and sharp is mocked before resizeAndSave.js loads. +import { resizeAndSave } from './resizeAndSave.js'; + +// resizeAndSave.js reads the bucket name once, when it loads. +vi.hoisted(() => { + process.env.ORIGINAL_BUCKET = 'test-bucket'; +}); + +// Mock sharp, so that these tests can make the resize succeed or fail on demand. +// These tests cover our own fallback logic, not sharp's image processing. +// getOrCreateObject.test.js still runs the real sharp library against real image bytes. +const { sharpMock, sharpPipeline } = vi.hoisted(() => { + // sharp's resize() and withMetadata() return the pipeline, so that calls can be chained. + const pipeline = {}; + pipeline.resize = vi.fn(() => pipeline); + pipeline.withMetadata = vi.fn(() => pipeline); + pipeline.toBuffer = vi.fn(async () => Buffer.from('resized-image-bytes')); + + const sharpFn = vi.fn(() => pipeline); + // resizeAndSave reads sharp.position when a crop is requested. + sharpFn.position = { + top: 1, right: 2, bottom: 3, left: 4, centre: 5, + }; + + return { sharpMock: sharpFn, sharpPipeline: pipeline }; +}); + +vi.mock('sharp', () => ({ default: sharpMock })); + +// Mock the s3 client. +const s3Mock = mockClient(S3Client); + +// A size match for a 150x150 gif, in the shape the regex in getOrCreateObject produces. +const sizeMatch = ['-150x150.gif', '150', '150', 'gif']; + +// Build a getObject response, with the image bytes as a readable stream. +const s3Response = (bytes) => ({ + Body: Readable.from(Buffer.from(bytes)), + ContentType: 'image/gif', +}); + +describe('resizeAndSave', () => { + beforeEach(() => { + s3Mock.reset(); + s3Mock.on(PutObjectCommand).resolves({}); + vi.clearAllMocks(); + }); + + it('should resize the image, save it to S3, and return the sharp pipeline', async () => { + const result = await resizeAndSave( + s3Response('original-image-bytes'), + '/www.bu.edu/somesite/files/01/example.gif', + sizeMatch, + false, + ); + + // The original bytes and the animated flag go to sharp, + // and the dimensions parsed from the size match go to resize. + expect(sharpMock).toHaveBeenCalledWith(Buffer.from('original-image-bytes'), { animated: true }); + expect(sharpPipeline.resize).toHaveBeenCalledWith({ width: 150, height: 150 }); + + // The resized bytes are saved for future requests, under the size suffixed key. + const putCalls = s3Mock.commandCalls(PutObjectCommand); + expect(putCalls).toHaveLength(1); + expect(putCalls[0].args[0].input).toEqual({ + Bucket: 'test-bucket', + Key: 'rendered_media/www.bu.edu/somesite/files/01/example-150x150.gif', + Body: Buffer.from('resized-image-bytes'), + ContentType: 'image/gif', + Metadata: { + 'original-key': 'original_media/www.bu.edu/somesite/files/01/example.gif', + }, + }); + + // The pipeline itself is returned, for the caller to use as the response body. + expect(result).toBe(sharpPipeline); + }); + + it('should return the original image and save nothing when it exceeds the pixel limit', async () => { + // This is the error sharp throws for an image with more pixels than its limit. + // Animated images can hit it because every frame counts toward the limit. + sharpPipeline.toBuffer.mockRejectedValueOnce(new Error('Input image exceeds pixel limit')); + + const result = await resizeAndSave( + s3Response('too-many-pixels'), + '/www.bu.edu/somesite/files/01/huge.gif', + sizeMatch, + false, + ); + + // The original, unresized bytes are returned. + expect(Buffer.isBuffer(result)).toBe(true); + expect(result.toString()).toBe('too-many-pixels'); + + // Nothing is saved to S3, because there is no resized image to save. + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('should rethrow any other resize failure', async () => { + sharpPipeline.toBuffer.mockRejectedValueOnce(new Error('Input buffer contains unsupported image format')); + + await expect(resizeAndSave( + s3Response('not-an-image'), + '/www.bu.edu/somesite/files/01/broken.gif', + sizeMatch, + false, + )).rejects.toThrow('unsupported image format'); + + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('should resize with the crop position and include the crop in the saved key', async () => { + await resizeAndSave( + s3Response('original-image-bytes'), + '/www.bu.edu/somesite/files/01/example.gif', + sizeMatch, + 'top', + ); + + expect(sharpPipeline.resize).toHaveBeenCalledWith({ + width: 150, + height: 150, + fit: 'cover', + position: sharpMock.position.top, + }); + + expect(s3Mock.commandCalls(PutObjectCommand)[0].args[0].input.Key) + .toBe('rendered_media/www.bu.edu/somesite/files/01/example-150x150*crop-top.gif'); + }); +}); + +// This is the guard for the message matched in resizeAndSave. +// The message comes from sharp's native code, so a sharp upgrade could change it. +// limitInputPixels provokes the same error from a tiny image, with no large fixture needed. +describe('the sharp pixel limit error', () => { + it('should still contain the message that resizeAndSave matches on', async () => { + const { default: realSharp } = await vi.importActual('sharp'); + + const tinyPng = await realSharp({ + create: { + width: 4, height: 4, channels: 3, background: { r: 255, g: 0, b: 0 }, + }, + }).png().toBuffer(); + + await expect(realSharp(tinyPng, { animated: true, limitInputPixels: 1 }) + .resize({ width: 2, height: 2 }) + .withMetadata() + .toBuffer()).rejects.toThrow('exceeds pixel limit'); + }); +}); diff --git a/src/package.json b/src/package.json index fb142f2..005c255 100644 --- a/src/package.json +++ b/src/package.json @@ -1,6 +1,6 @@ { "name": "s3-object-lambda-app", - "version": "1.0.7", + "version": "1.0.8", "description": "", "main": "app.js", "type": "module",