Webp and animated media

.gitignore

@@ -2,3 +2,4 @@
 .DS_Store
 dist
 node_modules
+staging

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bu-protected-s3-object-lambda",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Delivers asset from S3 through an object lambda with access restrictions and image resizing",
   "type": "module",
   "scripts": {

src/getOrCreateObject/getOrCreateObject.js

@@ -26,7 +26,7 @@ async function getOrCreateObject(userRequest, domain) {
   const { url } = userRequest;
   const { pathname, searchParams } = new URL(url);
   // Get the size match from the pathname.
-  const sizeMatch = pathname.match(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif)$/);
+  const sizeMatch = pathname.match(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif|webp)$/);
 
   // Decode the pathname for unicode characters.
   const decodedPathname = decodeURIComponent(pathname);
@@ -67,7 +67,7 @@ async function getOrCreateObject(userRequest, domain) {
   // if the image is not found, and there is a size match, then resize the image and save it to S3.
   if (response.Code === 'NoSuchKey' && sizeMatch) {
     // Reconstruct what the original image s3 key would be, by removing the image size from the URL
-    const originalPath = decodedPathname.replace(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif)$/, '.$3');
+    const originalPath = decodedPathname.replace(/-(\d+)x(\d+)\.(jpg|jpeg|png|gif|webp)$/, '.$3');
     const originalKey = `${ORIGINAL_PATH_ROOT}/${domain}${originalPath}`;
 
     const originalResponse = await tryGetObject(userRequest, originalKey);

src/getOrCreateObject/getOrCreateObject.test.js

@@ -15,6 +15,10 @@ process.env.ORIGINAL_BUCKET = 'test-bucket';
 // so that the sharp library has something to resize.
 const singlePixelJpgReadable = Readable.from(Buffer.from('/9j/4AAQSkZJRgABAQEAAAAAAAD/2wBDAAoHBwkHBgoJCAkLCwoMDxkQDw4ODx4WFxIZJCAmJSMgIyIoLTkwKCo2KyIjMkQyNjs9QEBAJjBGS0U+Sjk/QD3/2wBDAQsLCw8NDx0QEB09KSMpPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT3/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9/KKKKAP/2Q==', 'base64'));
 
+// Create a 1 pixel webp image as a readable stream,
+// so that the sharp library has something to resize.
+const singlePixelWebpReadable = Readable.from(Buffer.from('UklGRjwAAABXRUJQVlA4IDAAAADQAQCdASoBAAEAAUAmJaACdLoB+AADsAD+8ut//NgVzXPv9//S4P0uD9Lg/9KQAAA=', 'base64'));
+
 // Mock the ddb client.
 const ddbMock = mockClient(DynamoDBDocumentClient);
 ddbMock.on(GetCommand).resolves({
@@ -54,6 +58,14 @@ s3Mock.on(GetObjectCommand, {
   Body: singlePixelJpgReadable,
 });
 
+// Define an existing webp object in the bucket, that returns a valid and rescalable image.
+s3Mock.on(GetObjectCommand, {
+  Bucket: 'test-bucket',
+  Key: 'original_media/www.bu.edu/somesite/files/01/exists.webp',
+}).resolves({
+  Body: singlePixelWebpReadable,
+});
+
 describe('getOrCreateObject', () => {
   it('should return an object if it exists', async () => {
     const result = await getOrCreateObject(
@@ -99,6 +111,17 @@ describe('getOrCreateObject', () => {
     expect(result.Code).toEqual('NoSuchKey');
   });
 
+  it('should find the unscaled webp original for the request, scale it, and return an object', async () => {
+    const result = await getOrCreateObject(
+      {
+        url: 'https://example-1111.s3-object-lambda.us-east-1.amazonaws.com/somesite/files/01/exists-300x200.webp',
+        headers: { },
+      },
+      'www.bu.edu',
+    );
+    expect(result.Body).toBeDefined();
+  });
+
   it('should return a pre-scaled original object if it exists', async () => {
     const result = await getOrCreateObject(
       {

src/getOrCreateObject/resizeAndSave.js

@@ -13,18 +13,12 @@ const bucketName = process.env.ORIGINAL_BUCKET;
 
 const s3 = new S3();
 
-function getOriginalS3Key(url) {
-  // Reconstruct what the original image s3 key would be, by removing the image size from the URL.
-  const originalUrl = url.replace(/-(\d+)x(\d+)\.(jpg|png)$/, '.$3');
-  const parsedUrl = new URL(originalUrl);
-  const { pathname } = parsedUrl;
-  // The s3 key is the pathname without the leading slash.
-  const s3Key = pathname.replace(/^\//, '');
-
-  return s3Key;
-}
+// Sharp won't decode an image with more pixels than its limit, 268402689 pixels by default.
+// Animated images count every frame toward that limit, so a long animation can exceed it.
+const PIXEL_LIMIT_MESSAGE = 'exceeds pixel limit';
 
 // Resize and save the image to S3, then return the resized image data.
+// If the image is too large for sharp to resize, return the original image data instead.
 async function resizeAndSave(data, originalPath, sizeMatch, crop) {
   // Get the width and height from the sizeMatch as integers.
   const width = parseInt(sizeMatch[1], 10);
@@ -41,19 +35,33 @@ async function resizeAndSave(data, originalPath, sizeMatch, crop) {
   // This used to not be necessary but changed with the v3 S3 SDK.
   const imageBuffer = await streamToString(data.Body);
 
-  // Resize the image data with sharp.
-  const resized = await sharp(imageBuffer).resize({
-    width,
-    height,
-    ...options,
-  }).withMetadata();
+  // Resize the image data with sharp, and get the resized image data as a buffer.
+  let resized;
+  let resizedBuffer;
+  try {
+    resized = await sharp(imageBuffer, { animated: true }).resize({
+      width,
+      height,
+      ...options,
+    }).withMetadata();
+
+    resizedBuffer = await resized.toBuffer();
+  } catch (error) {
+    // Anything other than the pixel limit is unexpected, so let it propagate.
+    if (!error?.message?.includes(PIXEL_LIMIT_MESSAGE)) {
+      throw error;
+    }
+
+    // This image can't be resized, so serve the original unscaled for this request.
+    // Nothing is saved to S3, because there is no resized image to save.
+    console.log('Resize skipped, the image exceeds the sharp pixel limit: ', originalPath);
+
+    return imageBuffer;
+  }
 
   // Strip file extension from the original s3Key.
   const pathWithoutExtension = originalPath.replace(/\.[^/.]+$/, '');
 
-  // Get the resized image data as a buffer.
-  const resizedBuffer = await resized.toBuffer();
-
   // Encode the original path, so that it can be used in the metadata.
   const encodedPath = encodeURI(originalPath);
 
@@ -71,4 +79,4 @@ async function resizeAndSave(data, originalPath, sizeMatch, crop) {
   return resized;
 }
 
-export { getOriginalS3Key, resizeAndSave };
+export { resizeAndSave };

src/getOrCreateObject/resizeAndSave.test.js

@@ -0,0 +1,160 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import {
+  describe, it, expect, vi, beforeEach,
+} from 'vitest';
+import { mockClient } from 'aws-sdk-client-mock';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+
+// Import the function to test.
+// Vitest hoists the vi.hoisted() and vi.mock() calls below above this import,
+// so the bucket name is set and sharp is mocked before resizeAndSave.js loads.
+import { resizeAndSave } from './resizeAndSave.js';
+
+// resizeAndSave.js reads the bucket name once, when it loads.
+vi.hoisted(() => {
+  process.env.ORIGINAL_BUCKET = 'test-bucket';
+});
+
+// Mock sharp, so that these tests can make the resize succeed or fail on demand.
+// These tests cover our own fallback logic, not sharp's image processing.
+// getOrCreateObject.test.js still runs the real sharp library against real image bytes.
+const { sharpMock, sharpPipeline } = vi.hoisted(() => {
+  // sharp's resize() and withMetadata() return the pipeline, so that calls can be chained.
+  const pipeline = {};
+  pipeline.resize = vi.fn(() => pipeline);
+  pipeline.withMetadata = vi.fn(() => pipeline);
+  pipeline.toBuffer = vi.fn(async () => Buffer.from('resized-image-bytes'));
+
+  const sharpFn = vi.fn(() => pipeline);
+  // resizeAndSave reads sharp.position when a crop is requested.
+  sharpFn.position = {
+    top: 1, right: 2, bottom: 3, left: 4, centre: 5,
+  };
+
+  return { sharpMock: sharpFn, sharpPipeline: pipeline };
+});
+
+vi.mock('sharp', () => ({ default: sharpMock }));
+
+// Mock the s3 client.
+const s3Mock = mockClient(S3Client);
+
+// A size match for a 150x150 gif, in the shape the regex in getOrCreateObject produces.
+const sizeMatch = ['-150x150.gif', '150', '150', 'gif'];
+
+// Build a getObject response, with the image bytes as a readable stream.
+const s3Response = (bytes) => ({
+  Body: Readable.from(Buffer.from(bytes)),
+  ContentType: 'image/gif',
+});
+
+describe('resizeAndSave', () => {
+  beforeEach(() => {
+    s3Mock.reset();
+    s3Mock.on(PutObjectCommand).resolves({});
+    vi.clearAllMocks();
+  });
+
+  it('should resize the image, save it to S3, and return the sharp pipeline', async () => {
+    const result = await resizeAndSave(
+      s3Response('original-image-bytes'),
+      '/www.bu.edu/somesite/files/01/example.gif',
+      sizeMatch,
+      false,
+    );
+
+    // The original bytes and the animated flag go to sharp,
+    // and the dimensions parsed from the size match go to resize.
+    expect(sharpMock).toHaveBeenCalledWith(Buffer.from('original-image-bytes'), { animated: true });
+    expect(sharpPipeline.resize).toHaveBeenCalledWith({ width: 150, height: 150 });
+
+    // The resized bytes are saved for future requests, under the size suffixed key.
+    const putCalls = s3Mock.commandCalls(PutObjectCommand);
+    expect(putCalls).toHaveLength(1);
+    expect(putCalls[0].args[0].input).toEqual({
+      Bucket: 'test-bucket',
+      Key: 'rendered_media/www.bu.edu/somesite/files/01/example-150x150.gif',
+      Body: Buffer.from('resized-image-bytes'),
+      ContentType: 'image/gif',
+      Metadata: {
+        'original-key': 'original_media/www.bu.edu/somesite/files/01/example.gif',
+      },
+    });
+
+    // The pipeline itself is returned, for the caller to use as the response body.
+    expect(result).toBe(sharpPipeline);
+  });
+
+  it('should return the original image and save nothing when it exceeds the pixel limit', async () => {
+    // This is the error sharp throws for an image with more pixels than its limit.
+    // Animated images can hit it because every frame counts toward the limit.
+    sharpPipeline.toBuffer.mockRejectedValueOnce(new Error('Input image exceeds pixel limit'));
+
+    const result = await resizeAndSave(
+      s3Response('too-many-pixels'),
+      '/www.bu.edu/somesite/files/01/huge.gif',
+      sizeMatch,
+      false,
+    );
+
+    // The original, unresized bytes are returned.
+    expect(Buffer.isBuffer(result)).toBe(true);
+    expect(result.toString()).toBe('too-many-pixels');
+
+    // Nothing is saved to S3, because there is no resized image to save.
+    expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0);
+  });
+
+  it('should rethrow any other resize failure', async () => {
+    sharpPipeline.toBuffer.mockRejectedValueOnce(new Error('Input buffer contains unsupported image format'));
+
+    await expect(resizeAndSave(
+      s3Response('not-an-image'),
+      '/www.bu.edu/somesite/files/01/broken.gif',
+      sizeMatch,
+      false,
+    )).rejects.toThrow('unsupported image format');
+
+    expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0);
+  });
+
+  it('should resize with the crop position and include the crop in the saved key', async () => {
+    await resizeAndSave(
+      s3Response('original-image-bytes'),
+      '/www.bu.edu/somesite/files/01/example.gif',
+      sizeMatch,
+      'top',
+    );
+
+    expect(sharpPipeline.resize).toHaveBeenCalledWith({
+      width: 150,
+      height: 150,
+      fit: 'cover',
+      position: sharpMock.position.top,
+    });
+
+    expect(s3Mock.commandCalls(PutObjectCommand)[0].args[0].input.Key)
+      .toBe('rendered_media/www.bu.edu/somesite/files/01/example-150x150*crop-top.gif');
+  });
+});
+
+// This is the guard for the message matched in resizeAndSave.
+// The message comes from sharp's native code, so a sharp upgrade could change it.
+// limitInputPixels provokes the same error from a tiny image, with no large fixture needed.
+describe('the sharp pixel limit error', () => {
+  it('should still contain the message that resizeAndSave matches on', async () => {
+    const { default: realSharp } = await vi.importActual('sharp');
+
+    const tinyPng = await realSharp({
+      create: {
+        width: 4, height: 4, channels: 3, background: { r: 255, g: 0, b: 0 },
+      },
+    }).png().toBuffer();
+
+    await expect(realSharp(tinyPng, { animated: true, limitInputPixels: 1 })
+      .resize({ width: 2, height: 2 })
+      .withMetadata()
+      .toBuffer()).rejects.toThrow('exceeds pixel limit');
+  });
+});

src/package.json

@@ -1,6 +1,6 @@
 {
     "name": "s3-object-lambda-app",
-    "version": "1.0.7",
+    "version": "1.0.8",
     "description": "",
     "main": "app.js",
     "type": "module",