diff --git a/src/ebrains/+ebrains/+bucket/+api/BucketsClient.m b/src/ebrains/+ebrains/+bucket/+api/BucketsClient.m index 3f9aa29f..21128f9a 100644 --- a/src/ebrains/+ebrains/+bucket/+api/BucketsClient.m +++ b/src/ebrains/+ebrains/+bucket/+api/BucketsClient.m @@ -3,12 +3,14 @@ % % Each method mirrors one endpoint under /buckets of the Data Proxy API: % bucket stat, one page of the object listing, temporary download and -% upload URLs, the three requests of a multipart upload, rename, and -% delete. +% upload URLs, the three requests of a multipart upload, rename, copy, +% and delete. % An object name that holds "/" (a folder within the bucket) is sent with % the "/" as path separators, since the proxy takes the rest of the path -% as the name. The temporary URLs the proxy returns are made valid URLs -% before they are handed back; see the local functions in this file. +% as the name. The copy endpoint is the exception: copyObject sends the +% name as one segment with every "/" encoded, which its folder form +% needs. The temporary URLs the proxy returns are made valid URLs before +% they are handed back; see the local functions in this file. % % The functions of the ebrains.bucket namespace build on this client and % are the intended entry point. Use the client directly when an endpoint @@ -19,7 +21,7 @@ % bucketStat = client.getBucketStat("my-bucket"); % % See also ebrains.bucket.listBucketObjects, ebrains.bucket.getBucketObject, -% ebrains.bucket.renameObject, ebrains.bucket.deleteObject +% ebrains.bucket.renameObject, ebrains.bucket.copyObject, ebrains.bucket.deleteObject properties (Constant, Access = protected) ErrorIdPrefix = "EBRAINS:Bucket" @@ -258,18 +260,24 @@ function completeMultipartUpload(obj, bucketName, objectName, uploadId, partETag end end - function renameObject(obj, bucketName, objectName, targetName) + function renameObject(obj, bucketName, objectName, targetName, options) % renameObject - Rename an object of a bucket % % client.renameObject(bucketName, objectName, targetName) gives % the object the name targetName. To rename a folder, end - % objectName with "/". + % objectName with "/". The proxy queues the rename and completes + % it in the background; an object of the target name is replaced. + % + % client.renameObject(..., SendSuccessEmail=true) lets the proxy + % mail the account once the rename is done. The default is false. + % A rename that fails is reported by mail either way. arguments obj (1,1) ebrains.bucket.api.BucketsClient bucketName (1,1) string {mustBeNonzeroLengthText} objectName (1,1) string {mustBeNonzeroLengthText} targetName (1,1) string {mustBeNonzeroLengthText} + options.SendSuccessEmail (1,1) logical = false end % The target travels in the body, where nothing changes it, but @@ -282,15 +290,75 @@ function renameObject(obj, bucketName, objectName, targetName) end payload = struct('rename', struct('target_name', targetName)); + requiredParams = struct('send_success_email', options.SendSuccessEmail); - apiUri = obj.buildApiUri(["buckets", bucketName, objectPathSegments(objectName)]); + apiUri = obj.buildApiUri(["buckets", bucketName, objectPathSegments(objectName)], requiredParams); request = obj.initializeRequestMessage("PATCH", JSONPayload=jsonencode(payload)); response = obj.sendRequest(request, apiUri); - if response.StatusCode ~= "OK" + if ~isQueuedOrDone(response) obj.throwError("renameObject", response) end end + + function copyObject(obj, bucketName, objectName, targetName, options) + % copyObject - Copy an object of a bucket to another name in the same bucket + % + % client.copyObject(bucketName, objectName, targetName) writes a + % copy of the object under the name targetName and leaves the + % object itself in place. To copy a folder with everything in it, + % end objectName and targetName with "/". The proxy queues the + % copy and completes it in the background; an object of the + % target name is replaced. + % + % client.copyObject(..., SendSuccessEmail=true) lets the proxy + % mail the account once the copy is done. The default is false. + % A copy that fails is reported by mail either way. + + arguments + obj (1,1) ebrains.bucket.api.BucketsClient + bucketName (1,1) string {mustBeNonzeroLengthText} + objectName (1,1) string {mustBeNonzeroLengthText} + targetName (1,1) string {mustBeNonzeroLengthText} + options.SendSuccessEmail (1,1) logical = false + end + + % Neither name goes through the path check of buildApiUri: the + % source is sent as one segment (see below) and the target in + % the query. An object with such a name could not be reached + % afterwards, so both are refused here. + for name = [objectName, targetName] + if hasDotSegment(split(name, "/")) + error('EBRAINS:Bucket:DotSegmentInName', ... + ['The name "%s" has a "." or ".." segment, so the copied object could not be ' ... + 'reached by its name. Give names without "." or ".." between slashes.'], name) + end + end + + requiredParams = struct('name', targetName, 'send_success_email', options.SendSuccessEmail); + + % The endpoint is a PUT without a body, which MATLAB warns + % about on purpose. The warning is off for this request only. + warnState = warning('off', 'MATLAB:http:BodyExpectedFor'); + warningCleanup = onCleanup(@() warning(warnState)); + + % The name is sent as a single segment, with every "/" encoded + % as %2F, unlike for the other endpoints (see objectPathSegments). + % A trailing "/" marks a folder, and sent as a path separator it + % would stand right before the "copy" segment, where the proxy + % collapses the two slashes into one before it reads the name: + % the copy then runs on an object named like the folder, which + % does not exist, and fails. The copy endpoint signs no URL + % with the name, so the encoded form reaches it intact. + apiUri = obj.buildApiUri(["buckets", bucketName, objectName, "copy"], requiredParams); + request = obj.initializeRequestMessage("PUT"); + response = obj.sendRequest(request, apiUri); + + if ~isQueuedOrDone(response) + obj.throwError("copyObject", response) + end + end + function deleteObject(obj, bucketName, objectName) % deleteObject - Delete an object of a bucket % @@ -357,6 +425,15 @@ function deleteObject(obj, bucketName, objectName) tf = any(segments == "." | segments == ".."); end +function tf = isQueuedOrDone(response) +% isQueuedOrDone - Whether the reply to a rename or copy means the proxy took the request +% +% The proxy answers 201 ("Your copy is processing") when it has queued +% the operation to complete in the background, and 200 when it is done +% within the request. Both mean the request was accepted. + tf = response.StatusCode == "OK" || response.StatusCode == "Created"; +end + function segments = objectPathSegments(objectName) % objectPathSegments - Path segments of an object name, one per folder level % diff --git a/src/ebrains/+ebrains/+bucket/+internal/checkSourceAndTarget.m b/src/ebrains/+ebrains/+bucket/+internal/checkSourceAndTarget.m new file mode 100644 index 00000000..c448f4b2 --- /dev/null +++ b/src/ebrains/+ebrains/+bucket/+internal/checkSourceAndTarget.m @@ -0,0 +1,61 @@ +function checkSourceAndTarget(client, bucketName, objectName, targetName, overwrite) +% checkSourceAndTarget - Refuse a rename or copy that the proxy would accept and then fail or misapply +% +% ebrains.bucket.internal.checkSourceAndTarget(client, bucketName, +% objectName, targetName, overwrite) throws +% EBRAINS:Bucket:SameName when the two names are equal +% EBRAINS:Bucket:FolderNameMismatch when one name ends with "/" +% (a folder) and the other does not +% EBRAINS:Bucket:TargetWithinSource when the target lies within the +% source folder, so the folder +% would go into itself +% EBRAINS:Bucket:ObjectNotFound when the bucket holds no object +% named objectName +% EBRAINS:Bucket:TargetExists when it holds one named +% targetName and overwrite is false +% A folder counts as present when any object is named below it. +% +% The Data Proxy queues a rename or copy and reports one that fails +% only by mail, so these checks are the only answer the caller gets at +% once. The name checks send nothing; the other two send listing +% requests, never the rename or copy. +% +% See also ebrains.bucket.internal.hasObject + + arguments + client (1,1) ebrains.bucket.api.BucketsClient + bucketName (1,1) string {mustBeNonzeroLengthText} + objectName (1,1) string {mustBeNonzeroLengthText} + targetName (1,1) string {mustBeNonzeroLengthText} + overwrite (1,1) logical + end + + if objectName == targetName + error('EBRAINS:Bucket:SameName', ... + 'The target name "%s" is the name of the object itself. Give another target name.', targetName) + end + + isFolder = endsWith(objectName, "/"); + if isFolder ~= endsWith(targetName, "/") + error('EBRAINS:Bucket:FolderNameMismatch', ... + ['The names "%s" and "%s" must both end with "/", for a folder, or neither, for an ' ... + 'object.'], objectName, targetName) + end + + if isFolder && startsWith(targetName, objectName) + error('EBRAINS:Bucket:TargetWithinSource', ... + 'The target "%s" lies within the folder "%s", which would go into itself.', ... + targetName, objectName) + end + + if ~ebrains.bucket.internal.hasObject(client, bucketName, objectName) + error('EBRAINS:Bucket:ObjectNotFound', ... + 'Object "%s" was not found in bucket "%s".', objectName, bucketName) + end + + if ~overwrite && ebrains.bucket.internal.hasObject(client, bucketName, targetName) + error('EBRAINS:Bucket:TargetExists', ... + ['An object named "%s" is already in bucket "%s". Pass Overwrite=true to replace it, ' ... + 'or give another target name.'], targetName, bucketName) + end +end diff --git a/src/ebrains/+ebrains/+bucket/+internal/hasObject.m b/src/ebrains/+ebrains/+bucket/+internal/hasObject.m new file mode 100644 index 00000000..7b264c93 --- /dev/null +++ b/src/ebrains/+ebrains/+bucket/+internal/hasObject.m @@ -0,0 +1,38 @@ +function tf = hasObject(client, bucketName, objectName) +% hasObject - Whether a bucket holds an object of a name, or any object below a folder name +% +% tf = ebrains.bucket.internal.hasObject(client, bucketName, objectName) +% lists the bucket with the name as prefix and returns true when an +% object has exactly that name. The prefix also matches longer names +% ("a.txt" and "a.txt.bak"), hence the exact-name comparison. +% +% A name that ends with "/" stands for a folder, and the function +% returns true when any object is named below it. The listing is +% narrowed to one object then, since one is enough to answer. +% +% See also ebrains.bucket.renameObject, ebrains.bucket.copyObject + + arguments + client (1,1) ebrains.bucket.api.BucketsClient + bucketName (1,1) string {mustBeNonzeroLengthText} + objectName (1,1) string {mustBeNonzeroLengthText} + end + + isFolderName = endsWith(objectName, "/"); + if isFolderName + page = client.listObjects(bucketName, prefix=objectName, limit=1); + else + page = client.listObjects(bucketName, prefix=objectName); + end + objects = page.objects; + + % jsondecode turns an empty JSON array into [], which has no name field, + % so an empty listing cannot go through the name comparison. + if isempty(objects) + tf = false; + elseif isFolderName + tf = true; + else + tf = any(strcmp({objects.name}, objectName)); + end +end diff --git a/src/ebrains/+ebrains/+bucket/copyObject.m b/src/ebrains/+ebrains/+bucket/copyObject.m new file mode 100644 index 00000000..a5c054d8 --- /dev/null +++ b/src/ebrains/+ebrains/+bucket/copyObject.m @@ -0,0 +1,63 @@ +function copyObject(bucketName, objectName, targetName, options) +% copyObject - Copy an object (file) of a Data Proxy bucket to another name +% +% Syntax: +% ebrains.bucket.copyObject(bucketName, objectName, targetName) +% writes a copy of the object under the name targetName, in the same +% bucket, and leaves the object itself in place. The Data Proxy +% copies the object within the object store, so no data passes +% through MATLAB. Both names are relative to the root of the bucket; +% a leading "/" is ignored. To copy a folder with everything in it, +% end both names with "/". +% +% The Data Proxy queues the copy and completes it in the background, +% and reports a copy that fails only by mail. So before the request, +% the function checks that the object is in the bucket (error +% EBRAINS:Bucket:ObjectNotFound otherwise) and, unless Overwrite is +% true, that no object has the target name (error +% EBRAINS:Bucket:TargetExists otherwise). A folder is present when +% any object is named below it. Two equal names, a folder name +% paired with an object name, and a target within the source folder +% are refused as well. No check sends the copy, so a refused copy +% produces no mail. The copy appears in the listing once it has +% completed. +% +% ebrains.bucket.copyObject(..., Name=Value) sets the options below. +% +% Input Arguments +% bucketName : Name of the bucket that holds the object +% objectName : Name of the object to copy +% targetName : Name of the copy +% +% Name-Value Arguments +% Overwrite : Replace an object that has the target name +% rather than refuse the copy. Default is false. +% SendSuccessEmail : Let the Data Proxy mail the account once the +% copy is done. Default is false; a copy that +% fails is reported by mail either way. +% Client : ebrains.bucket.api.BucketsClient that sends the +% requests. Meant for tests and custom clients; a +% default client is created otherwise. +% +% See also ebrains.bucket.renameObject, ebrains.bucket.deleteObject + + arguments + bucketName (1,1) string + objectName (1,1) string + targetName (1,1) string + options.Overwrite (1,1) logical = false + options.SendSuccessEmail (1,1) logical = false + options.Client (1,1) ebrains.bucket.api.BucketsClient = ebrains.bucket.api.BucketsClient() + end + + % Names are relative to the bucket root, so a leading "/" would be + % sent as an empty first folder. + objectName = ebrains.bucket.internal.removeLeadingSlash(objectName); + targetName = ebrains.bucket.internal.removeLeadingSlash(targetName); + + ebrains.bucket.internal.checkSourceAndTarget(options.Client, bucketName, objectName, targetName, ... + options.Overwrite); + + options.Client.copyObject(bucketName, objectName, targetName, ... + SendSuccessEmail=options.SendSuccessEmail); +end diff --git a/src/ebrains/+ebrains/+bucket/renameObject.m b/src/ebrains/+ebrains/+bucket/renameObject.m index f860c73e..dfb43e7d 100644 --- a/src/ebrains/+ebrains/+bucket/renameObject.m +++ b/src/ebrains/+ebrains/+bucket/renameObject.m @@ -3,8 +3,25 @@ function renameObject(bucketName, objectName, targetName, options) % % Syntax: % ebrains.bucket.renameObject(bucketName, objectName, targetName) -% gives the object the name targetName. Both names are relative to -% the root of the bucket; a leading "/" is ignored. +% gives the object the name targetName. The Data Proxy moves the +% object within the object store, so no data passes through MATLAB. +% Both names are relative to the root of the bucket; a leading "/" +% is ignored. To rename a folder with everything in it, end +% objectName with "/". +% +% The Data Proxy queues the rename and completes it in the +% background, and reports a rename that fails only by mail. So +% before the request, the function checks that the object is in the +% bucket (error EBRAINS:Bucket:ObjectNotFound otherwise) and, unless +% Overwrite is true, that no object has the target name (error +% EBRAINS:Bucket:TargetExists otherwise). A folder is present when +% any object is named below it. Two equal names, a folder name +% paired with an object name, and a target within the source folder +% are refused as well. No check sends the rename, so a refused +% rename produces no mail. The object can be listed under both +% names until the rename has completed. +% +% ebrains.bucket.renameObject(..., Name=Value) sets the options below. % % Input Arguments % bucketName : Name of the bucket that holds the object @@ -12,14 +29,23 @@ function renameObject(bucketName, objectName, targetName, options) % targetName : New name of the object % % Name-Value Arguments -% Client : ebrains.bucket.api.BucketsClient that sends the request. -% Meant for tests and custom clients; a default client is -% created otherwise. +% Overwrite : Replace an object that has the target name +% rather than refuse the rename. Default is false. +% SendSuccessEmail : Let the Data Proxy mail the account once the +% rename is done. Default is false; a rename +% that fails is reported by mail either way. +% Client : ebrains.bucket.api.BucketsClient that sends the +% requests. Meant for tests and custom clients; a +% default client is created otherwise. +% +% See also ebrains.bucket.copyObject, ebrains.bucket.deleteObject arguments bucketName (1,1) string objectName (1,1) string targetName (1,1) string + options.Overwrite (1,1) logical = false + options.SendSuccessEmail (1,1) logical = false options.Client (1,1) ebrains.bucket.api.BucketsClient = ebrains.bucket.api.BucketsClient() end @@ -28,5 +54,9 @@ function renameObject(bucketName, objectName, targetName, options) objectName = ebrains.bucket.internal.removeLeadingSlash(objectName); targetName = ebrains.bucket.internal.removeLeadingSlash(targetName); - options.Client.renameObject(bucketName, objectName, targetName); + ebrains.bucket.internal.checkSourceAndTarget(options.Client, bucketName, objectName, targetName, ... + options.Overwrite); + + options.Client.renameObject(bucketName, objectName, targetName, ... + SendSuccessEmail=options.SendSuccessEmail); end diff --git a/tests/+ebrains/+test/+bucket/+api/BucketsClientTest.m b/tests/+ebrains/+test/+bucket/+api/BucketsClientTest.m index 6c2519e3..1dc7673c 100644 --- a/tests/+ebrains/+test/+bucket/+api/BucketsClientTest.m +++ b/tests/+ebrains/+test/+bucket/+api/BucketsClientTest.m @@ -26,6 +26,7 @@ "getDownloadUrl", @(client, bucket, name) client.getDownloadUrl(bucket, name), ... "getUploadUrl", @(client, bucket, name) client.getUploadUrl(bucket, name), ... "renameObject", @(client, bucket, name) client.renameObject(bucket, name, "new.txt"), ... + "copyObject", @(client, bucket, name) client.copyObject(bucket, name, "new.txt"), ... "deleteObject", @(client, bucket, name) client.deleteObject(bucket, name)) % Each endpoint method called with a name that has a ".." segment endpointCallWithDotSegment = struct( ... @@ -34,6 +35,7 @@ "getDownloadUrl", @(client) client.getDownloadUrl("my-bucket", "a/../b.txt"), ... "getUploadUrl", @(client) client.getUploadUrl("my-bucket", "a/../b.txt"), ... "renameObject", @(client) client.renameObject("my-bucket", "a/../b.txt", "new.txt"), ... + "copyObject", @(client) client.copyObject("my-bucket", "a/../b.txt", "new.txt"), ... "deleteObject", @(client) client.deleteObject("my-bucket", "a/../b.txt")) end @@ -276,17 +278,124 @@ function testRenameObjectSendsPatchWithPayload(testCase) testCase.Client.renameObject("my-bucket", "old.txt", "new.txt"); testCase.Client.verifyRequestMethod(1, 'PATCH'); - testCase.Client.verifyRequestURL(1, '/api/v1/buckets/my-bucket/old.txt'); + testCase.Client.verifyRequestURL(1, '/api/v1/buckets/my-bucket/old.txt?'); testCase.verifyEqual(testCase.Client.getRequestPayload(1), ... '{"rename":{"target_name":"new.txt"}}'); end + function testRenameObjectTurnsOffTheSuccessMailUnlessAsked(testCase) + % The proxy mails the account for every finished rename unless + % the request says otherwise. + testCase.Client.addResponse('OK', struct()); + testCase.Client.addResponse('OK', struct()); + + testCase.Client.renameObject("my-bucket", "old.txt", "new.txt"); + testCase.Client.renameObject("my-bucket", "old.txt", "new.txt", SendSuccessEmail=true); + + testCase.Client.verifyRequestURL(1, 'send_success_email=0'); + testCase.Client.verifyRequestURL(2, 'send_success_email=1'); + end + + function testRenameObjectAcceptsQueuedRename(testCase) + % 201 means the proxy has queued the rename; it is accepted as + % 200 is. + testCase.Client.addResponse('Created', struct('status_code', 201, ... + 'detail', 'Your rename is processing.')); + + testCase.Client.renameObject("my-bucket", "old.txt", "new.txt"); + + testCase.verifyEqual(testCase.Client.getRequestCount(), 1); + end + function testRenameObjectValidationError(testCase) testCase.Client.addResponse('UnprocessableEntity', struct('detail', 'invalid name')); testCase.verifyError(@() testCase.Client.renameObject("my-bucket", "old.txt", "new.txt"), ... 'EBRAINS:Bucket:renameObject:UnprocessableEntity'); end + %% copyObject + function testCopyObjectSendsPutWithTargetInQueryAndNoBody(testCase) + testCase.Client.addResponse('OK', struct()); + + testCase.Client.copyObject("my-bucket", "sub dir/old.txt", "new dir/new.txt"); + + testCase.Client.verifyRequestMethod(1, 'PUT'); + testCase.Client.verifyRequestURL(1, '/api/v1/buckets/my-bucket/sub%20dir%2Fold.txt/copy?'); + testCase.Client.verifyRequestURL(1, 'name=new+dir/new.txt'); + request = testCase.Client.getRequest(1); + testCase.verifyEmpty(request.RequestMessage.Body); + end + + function testCopyObjectSendsFolderNameAsOneSegment(testCase) + % The trailing "/" of a folder name must reach the proxy as part + % of the name; as a path separator before "copy" it would be + % collapsed away and the request would copy a single object. + testCase.Client.addResponse('OK', struct()); + + testCase.Client.copyObject("my-bucket", "old folder/", "new folder/"); + + testCase.Client.verifyRequestURL(1, '/buckets/my-bucket/old%20folder%2F/copy?'); + testCase.Client.verifyRequestURL(1, 'name=new+folder/'); + end + + function testCopyObjectTurnsOffTheSuccessMailUnlessAsked(testCase) + % The proxy mails the account for every finished copy unless + % the request says otherwise. + testCase.Client.addResponse('OK', struct()); + testCase.Client.addResponse('OK', struct()); + + testCase.Client.copyObject("my-bucket", "old.txt", "new.txt"); + testCase.Client.copyObject("my-bucket", "old.txt", "new.txt", SendSuccessEmail=true); + + testCase.Client.verifyRequestURL(1, 'send_success_email=0'); + testCase.Client.verifyRequestURL(2, 'send_success_email=1'); + end + + function testCopyObjectAcceptsQueuedCopy(testCase) + % 201 means the proxy has queued the copy; it is accepted as + % 200 is. + testCase.Client.addResponse('Created', struct('status_code', 201, ... + 'detail', 'Your copy is processing.')); + + testCase.Client.copyObject("my-bucket", "old.txt", "new.txt"); + + testCase.verifyEqual(testCase.Client.getRequestCount(), 1); + end + + function testCopyObjectEncodesTheTargetInTheQuery(testCase) + % A "+" or "&" in a target name must not reach the proxy as a + % space or as a parameter separator. + testCase.Client.addResponse('OK', struct()); + + testCase.Client.copyObject("my-bucket", "old.txt", "a+b&c d.txt"); + + testCase.Client.verifyRequestURL(1, 'name=a%2Bb%26c+d.txt'); + end + + function testCopyObjectRestoresTheBodyWarning(testCase) + % The body-less PUT turns the warning off for the request only, + % also when the request ends in an error. + warningId = 'MATLAB:http:BodyExpectedFor'; + originalState = warning('query', warningId); + testCase.addTeardown(@() warning(originalState)); + warning('on', warningId); + testCase.Client.addResponse('OK', struct()); + testCase.Client.addResponse('NotFound', struct('detail', 'missing')); + + testCase.Client.copyObject("my-bucket", "old.txt", "new.txt"); + testCase.verifyEqual(warning('query', warningId).state, 'on'); + + testCase.verifyError(@() testCase.Client.copyObject("my-bucket", "old.txt", "new.txt"), ... + 'EBRAINS:Bucket:copyObject:NotFound'); + testCase.verifyEqual(warning('query', warningId).state, 'on'); + end + + function testCopyObjectValidationError(testCase) + testCase.Client.addResponse('UnprocessableEntity', struct('detail', 'invalid name')); + testCase.verifyError(@() testCase.Client.copyObject("my-bucket", "old.txt", "new.txt"), ... + 'EBRAINS:Bucket:copyObject:UnprocessableEntity'); + end + %% deleteObject function testDeleteObjectSendsDeleteWithoutBody(testCase) testCase.Client.addResponse('OK', struct()); @@ -338,6 +447,13 @@ function testRenameTargetWithDotSegmentIsRefused(testCase, dotSegmentName) testCase.verifyEqual(testCase.Client.getRequestCount(), 0); end + function testCopyTargetWithDotSegmentIsRefused(testCase, dotSegmentName) + testCase.verifyError( ... + @() testCase.Client.copyObject("my-bucket", "old.txt", dotSegmentName), ... + 'EBRAINS:Bucket:DotSegmentInName'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 0); + end + function testNameWithDotSegmentIsRefusedBeforeTheTokenLookup(testCase, endpointCallWithDotSegment) % The real client asks the token manager for the request header, % which can refresh a token or start a login, so a name that is diff --git a/tests/+ebrains/+test/+bucket/+internal/HasObjectTest.m b/tests/+ebrains/+test/+bucket/+internal/HasObjectTest.m new file mode 100644 index 00000000..c9544b0e --- /dev/null +++ b/tests/+ebrains/+test/+bucket/+internal/HasObjectTest.m @@ -0,0 +1,85 @@ +classdef HasObjectTest < matlab.unittest.TestCase + % HasObjectTest - Unit tests for ebrains.bucket.internal.hasObject + % + % Every call receives a MockBucketsClient, so no request reaches the + % Data Proxy. Page fixtures are column struct arrays, the shape + % jsondecode gives a JSON array, and [] for an empty one. + + properties + Client ebrains.mocks.MockBucketsClient + end + + methods (TestMethodSetup) + function createMockClient(testCase) + testCase.Client = ebrains.mocks.MockBucketsClient(); + end + end + + methods (Test) + function testEmptyListingMeansAbsent(testCase) + testCase.Client.addResponse('OK', makePage(string.empty)); + + tf = ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "a.txt"); + + testCase.verifyFalse(tf); + testCase.Client.verifyRequestURL(1, '/buckets/my-bucket?prefix=a.txt'); + end + + function testLongerNameOnlyMeansAbsent(testCase) + % The prefix listing also returns longer names, which must not + % count as the object. + testCase.Client.addResponse('OK', makePage("a.txt.bak")); + + tf = ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "a.txt"); + + testCase.verifyFalse(tf); + end + + function testExactNameAmongLongerOnesMeansPresent(testCase) + testCase.Client.addResponse('OK', makePage(["a.txt.bak", "a.txt"])); + + tf = ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "a.txt"); + + testCase.verifyTrue(tf); + end + + function testFolderIsPresentWhenAnObjectIsBelowIt(testCase) + % One listed object is enough to tell, so the listing asks for + % one. + testCase.Client.addResponse('OK', makePage("d/x.txt")); + + tf = ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "d/"); + + testCase.verifyTrue(tf); + testCase.Client.verifyRequestURL(1, 'prefix=d/'); + testCase.Client.verifyRequestURL(1, 'limit=1'); + end + + function testFolderWithNothingBelowIsAbsent(testCase) + testCase.Client.addResponse('OK', makePage(string.empty)); + + tf = ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "d/"); + + testCase.verifyFalse(tf); + end + + function testListingErrorPropagates(testCase) + testCase.Client.addResponse('NotFound', 'Bucket not found'); + + testCase.verifyError(... + @() ebrains.bucket.internal.hasObject(testCase.Client, "my-bucket", "a.txt"), ... + 'EBRAINS:Bucket:listObjects:NotFound'); + end + end +end + +function page = makePage(objectNames) +% makePage - Listing page as jsondecode returns it from the Data Proxy + if isempty(objectNames) + page = struct('objects', []); + return + end + names = cellstr(reshape(objectNames, [], 1)); + sizes = num2cell(zeros(numel(names), 1)); + page = struct('objects', struct('name', names, 'bytes', sizes)); +end diff --git a/tests/+ebrains/+test/+bucket/BucketFunctionsTest.m b/tests/+ebrains/+test/+bucket/BucketFunctionsTest.m index f8f5ce6b..a3eddc35 100644 --- a/tests/+ebrains/+test/+bucket/BucketFunctionsTest.m +++ b/tests/+ebrains/+test/+bucket/BucketFunctionsTest.m @@ -2,8 +2,10 @@ % BucketFunctionsTest - Unit tests for the functions of the ebrains.bucket namespace % % Every function receives a MockBucketsClient through its Client - % option, so no request reaches the Data Proxy. Page fixtures are - % column struct arrays, the shape jsondecode gives a JSON array. + % option, so no request reaches the Data Proxy. The mock answers the + % requests in the order the responses were queued, so response k is + % the reply to request k. Page fixtures are column struct arrays, the + % shape jsondecode gives a JSON array. properties Client ebrains.mocks.MockBucketsClient @@ -35,6 +37,14 @@ NestedObjectName = struct( ... "NestedName", "a/b.txt", ... "LeadingSlash", "/a/b.txt") + + % The functions that ask the proxy to rename or copy an object. + % Both list the source, then the target unless Overwrite is true, + % and send the request only when the source is there and the + % target is free. + moveFunction = struct( ... + "renameObject", @ebrains.bucket.renameObject, ... + "copyObject", @ebrains.bucket.copyObject) end methods (TestMethodSetup) @@ -190,24 +200,180 @@ function testListBucketObjectsStatErrorPropagates(testCase) 'EBRAINS:Bucket:getBucketStat:NotFound'); end - %% renameObject + %% renameObject and copyObject + % Both functions list the source name and the target name before + % the request, so the proxy is never asked to rename or copy what + % it would only report on by mail. The listing of a file name also + % returns longer names, which must not count as the object. function testRenameObjectIgnoresLeadingSlashes(testCase) - testCase.Client.addResponse('OK', struct()); + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the rename is accepted ebrains.bucket.renameObject("my-bucket", "/old.txt", "/sub/new.txt", Client=testCase.Client); - testCase.Client.verifyRequestURL(1, '/buckets/my-bucket/old.txt'); - testCase.verifyEqual(testCase.Client.getRequestPayload(1), ... + testCase.Client.verifyRequestURL(1, '/buckets/my-bucket?prefix=old.txt'); + testCase.Client.verifyRequestURL(2, '/buckets/my-bucket?prefix=sub/new.txt'); + testCase.Client.verifyRequestMethod(3, 'PATCH'); + testCase.Client.verifyRequestURL(3, '/buckets/my-bucket/old.txt?'); + testCase.verifyEqual(testCase.Client.getRequestPayload(3), ... '{"rename":{"target_name":"sub/new.txt"}}'); end function testRenameObjectErrorPropagates(testCase) - testCase.Client.addResponse('UnprocessableEntity', struct('detail', 'invalid')); + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('UnprocessableEntity', struct('detail', 'invalid')); % the rename is refused testCase.verifyError(... @() ebrains.bucket.renameObject("my-bucket", "old.txt", "new.txt", Client=testCase.Client), ... 'EBRAINS:Bucket:renameObject:UnprocessableEntity'); end + function testCopyObjectIgnoresLeadingSlashes(testCase) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the copy is accepted + + ebrains.bucket.copyObject("my-bucket", "/old.txt", "/sub/new.txt", Client=testCase.Client); + + testCase.Client.verifyRequestURL(1, '/buckets/my-bucket?prefix=old.txt'); + testCase.Client.verifyRequestURL(2, '/buckets/my-bucket?prefix=sub/new.txt'); + testCase.Client.verifyRequestMethod(3, 'PUT'); + testCase.Client.verifyRequestURL(3, '/buckets/my-bucket/old.txt/copy?'); + testCase.Client.verifyRequestURL(3, 'name=sub/new.txt'); + end + + function testCopyObjectErrorPropagates(testCase) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('UnprocessableEntity', struct('detail', 'invalid')); % the copy is refused + testCase.verifyError(... + @() ebrains.bucket.copyObject("my-bucket", "old.txt", "new.txt", Client=testCase.Client), ... + 'EBRAINS:Bucket:copyObject:UnprocessableEntity'); + end + + function testMoveRefusesAMissingSourceWithoutARequest(testCase, moveFunction) + testCase.Client.addResponse('OK', makePage("old.txt.bak")); % source listing: the source is missing + + testCase.verifyError(@() moveFunction("my-bucket", "old.txt", "new.txt", Client=testCase.Client), ... + 'EBRAINS:Bucket:ObjectNotFound'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 1); + end + + function testMoveRefusesATakenTargetWithoutARequest(testCase, moveFunction) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage("new.txt")); % target listing: the target is taken + + testCase.verifyError(@() moveFunction("my-bucket", "old.txt", "new.txt", Client=testCase.Client), ... + 'EBRAINS:Bucket:TargetExists'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 2); + end + + function testMoveTakesALongerNameForAFreeTarget(testCase, moveFunction) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage("new.txt.bak")); % target listing: a longer name only + testCase.Client.addResponse('OK', struct()); % the request is accepted + + moveFunction("my-bucket", "old.txt", "new.txt", Client=testCase.Client); + + testCase.verifyEqual(testCase.Client.getRequestCount(), 3); + end + + function testMoveWithOverwriteDoesNotListTheTarget(testCase, moveFunction) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', struct()); % the request is accepted + + moveFunction("my-bucket", "old.txt", "new.txt", Overwrite=true, Client=testCase.Client); + + testCase.verifyEqual(testCase.Client.getRequestCount(), 2); + testCase.Client.verifyRequestURL(2, '/buckets/my-bucket/old.txt'); + end + + function testMoveOfAFolderCountsAnyObjectBelowIt(testCase, moveFunction) + % A folder is present when any object is named below it; one + % listed object is enough to tell, so the listing asks for one. + testCase.Client.addResponse('OK', makePage("old/a.txt")); % source listing: an object below the folder + testCase.Client.addResponse('OK', makePage("new/b.txt")); % target listing: an object below the folder + + testCase.verifyError(@() moveFunction("my-bucket", "old/", "new/", Client=testCase.Client), ... + 'EBRAINS:Bucket:TargetExists'); + testCase.Client.verifyRequestURL(1, 'prefix=old/'); + testCase.Client.verifyRequestURL(1, 'limit=1'); + testCase.Client.verifyRequestURL(2, 'prefix=new/'); + end + + function testMoveRefusesTheSameNameWithoutARequest(testCase, moveFunction) + % Overwrite would skip the target check, so the names are + % compared before anything is listed. + testCase.verifyError(... + @() moveFunction("my-bucket", "a.txt", "/a.txt", Overwrite=true, Client=testCase.Client), ... + 'EBRAINS:Bucket:SameName'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 0); + end + + function testMoveRefusesAFolderPairedWithAnObjectName(testCase, moveFunction) + testCase.verifyError(... + @() moveFunction("my-bucket", "d/", "d2", Client=testCase.Client), ... + 'EBRAINS:Bucket:FolderNameMismatch'); + testCase.verifyError(... + @() moveFunction("my-bucket", "d", "d2/", Client=testCase.Client), ... + 'EBRAINS:Bucket:FolderNameMismatch'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 0); + end + + function testMoveRefusesATargetWithinTheSourceFolder(testCase, moveFunction) + testCase.verifyError(... + @() moveFunction("my-bucket", "a/", "a/b/", Overwrite=true, Client=testCase.Client), ... + 'EBRAINS:Bucket:TargetWithinSource'); + testCase.verifyEqual(testCase.Client.getRequestCount(), 0); + end + + function testMoveAllowsATargetAboveTheSourceFolder(testCase, moveFunction) + % Moving a folder up one level flattens it, which is allowed. + testCase.Client.addResponse('OK', makePage("a/b/x.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the request is accepted + + moveFunction("my-bucket", "a/b/", "c/", Client=testCase.Client); + + testCase.verifyEqual(testCase.Client.getRequestCount(), 3); + end + + function testRenameObjectSendsAFolderRequest(testCase) + testCase.Client.addResponse('OK', makePage("old folder/a.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the rename is accepted + + ebrains.bucket.renameObject("my-bucket", "old folder/", "new folder/", Client=testCase.Client); + + testCase.Client.verifyRequestMethod(3, 'PATCH'); + testCase.Client.verifyRequestURL(3, '/buckets/my-bucket/old%20folder/?'); + testCase.verifyEqual(testCase.Client.getRequestPayload(3), ... + '{"rename":{"target_name":"new folder/"}}'); + end + + function testCopyObjectSendsAFolderRequest(testCase) + testCase.Client.addResponse('OK', makePage("old folder/a.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the copy is accepted + + ebrains.bucket.copyObject("my-bucket", "old folder/", "new folder/", Client=testCase.Client); + + testCase.Client.verifyRequestMethod(3, 'PUT'); + testCase.Client.verifyRequestURL(3, '/buckets/my-bucket/old%20folder%2F/copy?'); + testCase.Client.verifyRequestURL(3, 'name=new+folder/'); + end + + function testMovePassesTheMailSwitch(testCase, moveFunction) + testCase.Client.addResponse('OK', makePage("old.txt")); % source listing: the source exists + testCase.Client.addResponse('OK', makePage(string.empty)); % target listing: the target is free + testCase.Client.addResponse('OK', struct()); % the request is accepted + + moveFunction("my-bucket", "old.txt", "new.txt", SendSuccessEmail=true, Client=testCase.Client); + + testCase.Client.verifyRequestURL(3, 'send_success_email=1'); + end + %% deleteObject function testDeleteObjectIgnoresLeadingSlash(testCase) testCase.Client.addResponse('OK', struct()); diff --git a/tests/+ebrains/+test/+live/BucketLiveTest.m b/tests/+ebrains/+test/+live/BucketLiveTest.m index c6082a71..53863253 100644 --- a/tests/+ebrains/+test/+live/BucketLiveTest.m +++ b/tests/+ebrains/+test/+live/BucketLiveTest.m @@ -2,7 +2,7 @@ % BucketLiveTest - Live checks of the ebrains.bucket functions % % Exercises listing, upload in one request and in parts, download, - % size lookup, and delete against the real EBRAINS Data Proxy API, + % size lookup, copy, and delete against the real EBRAINS Data Proxy API, % using a bucket the authenticated account has write access to. % Tagged "LiveIntegration"; excluded from the default test run (see % tools/tasks/testToolbox.m) and run on a schedule by @@ -50,6 +50,105 @@ function testUploadDownloadDeleteRoundTrip(testCase) 'EBRAINS:Bucket:ObjectNotFound'); end + function testCopyObjectLeavesTheSourceAndWritesAnEqualCopy(testCase) + folderFixture = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture); + sourceFile = fullfile(folderFixture.Folder, "probe.txt"); + fileId = fopen(sourceFile, "w"); + fprintf(fileId, "EBRAINS-MATLAB live copy test %s\n", string(datetime("now", TimeZone="UTC"))); + fclose(fileId); + sourceSizeBytes = dir(sourceFile).bytes; + + stamp = string(datetime("now", Format="yyyyMMdd-HHmmssSSS")); + objectName = "live-test/copy-source-" + stamp + ".txt"; + copyName = "live-test/copied/copy-target-" + stamp + ".txt"; + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, objectName)); + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, copyName)); + + ebrains.bucket.uploadFile(testCase.BucketName, objectName, sourceFile); + ebrains.bucket.copyObject(testCase.BucketName, objectName, copyName); + + % The proxy completes the copy in the background after it has + % answered, so the copy is awaited rather than expected at once. + testCase.assertTrue(waitForObject(testCase.BucketName, copyName), ... + 'The copy did not appear in the bucket within the wait.'); + testCase.verifyEqual(ebrains.bucket.getFileSize(testCase.BucketName, copyName), sourceSizeBytes, ... + 'The copy must have the size of the source.'); + testCase.verifyEqual(ebrains.bucket.getFileSize(testCase.BucketName, objectName), sourceSizeBytes, ... + 'The source must still be in the bucket after the copy.'); + + targetFile = fullfile(folderFixture.Folder, "downloaded.txt"); + ebrains.bucket.downloadFile(testCase.BucketName, copyName, targetFile); + testCase.verifyTrue(isequal(readBytes(targetFile), readBytes(sourceFile)), ... + 'The downloaded copy differs from the source file.'); + end + + function testCopyObjectCopiesAFolderWithEverythingInIt(testCase) + folderFixture = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture); + sourceFile = fullfile(folderFixture.Folder, "probe.txt"); + writelines("EBRAINS-MATLAB live folder copy test " + string(datetime("now", TimeZone="UTC")), sourceFile); + + stamp = string(datetime("now", Format="yyyyMMdd-HHmmssSSS")); + sourceFolder = "live-test/copy-folder-" + stamp + "/"; + targetFolder = "live-test/copy-folder-" + stamp + "-copy/"; + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, sourceFolder)); + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, targetFolder)); + + ebrains.bucket.uploadFile(testCase.BucketName, sourceFolder + "a.txt", sourceFile); + ebrains.bucket.uploadFile(testCase.BucketName, sourceFolder + "sub/b.txt", sourceFile); + + ebrains.bucket.copyObject(testCase.BucketName, sourceFolder, targetFolder); + + % The copy completes in the background; the nested object is + % the last to look for, since the copy covers the whole folder. + testCase.assertTrue(waitForObject(testCase.BucketName, targetFolder + "sub/b.txt"), ... + 'The nested copy did not appear in the bucket within the wait.'); + testCase.assertTrue(waitForObject(testCase.BucketName, targetFolder + "a.txt")); + testCase.verifyEqual(ebrains.bucket.getFileSize(testCase.BucketName, targetFolder + "sub/b.txt"), ... + dir(sourceFile).bytes); + testCase.verifyTrue(ebrains.bucket.internal.hasObject( ... + ebrains.bucket.api.BucketsClient(), testCase.BucketName, sourceFolder), ... + 'The source folder must still be in the bucket after the copy.'); + end + + function testRenameObjectMovesTheObjectAndRefusesATakenTarget(testCase) + folderFixture = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture); + sourceFile = fullfile(folderFixture.Folder, "probe.txt"); + fileId = fopen(sourceFile, "w"); + fprintf(fileId, "EBRAINS-MATLAB live rename test %s\n", string(datetime("now", TimeZone="UTC"))); + fclose(fileId); + sourceSizeBytes = dir(sourceFile).bytes; + + stamp = string(datetime("now", Format="yyyyMMdd-HHmmssSSS")); + objectName = "live-test/rename-source-" + stamp + ".txt"; + takenName = "live-test/rename-taken-" + stamp + ".txt"; + newName = "live-test/renamed/rename-target-" + stamp + ".txt"; + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, objectName)); + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, takenName)); + testCase.addTeardown(@() deleteIfPresent(testCase.BucketName, newName)); + + ebrains.bucket.uploadFile(testCase.BucketName, objectName, sourceFile); + ebrains.bucket.uploadFile(testCase.BucketName, takenName, sourceFile); + + % An object that has the target name stops the rename before + % the request. + testCase.verifyError( ... + @() ebrains.bucket.renameObject(testCase.BucketName, objectName, takenName), ... + 'EBRAINS:Bucket:TargetExists'); + + ebrains.bucket.renameObject(testCase.BucketName, objectName, newName); + + % The proxy completes the rename in the background after it has + % answered, so the renamed object is awaited. + testCase.assertTrue(waitForObject(testCase.BucketName, newName), ... + 'The renamed object did not appear in the bucket within the wait.'); + testCase.verifyEqual(ebrains.bucket.getFileSize(testCase.BucketName, newName), sourceSizeBytes); + testCase.assertTrue(waitForObject(testCase.BucketName, objectName, Present=false), ... + 'The object is still listed under its old name after the rename.'); + testCase.verifyError( ... + @() ebrains.bucket.renameObject(testCase.BucketName, objectName, newName), ... + 'EBRAINS:Bucket:ObjectNotFound'); + end + function testMultipartUploadRoundTrip(testCase) % Three parts, the last one smaller than the others. The % threshold is lowered so that a file of a few MiB takes the @@ -83,6 +182,29 @@ function testMultipartUploadRoundTrip(testCase) end end +function isSettled = waitForObject(bucketName, objectName, options) +% waitForObject - Poll the bucket until the object is present (or absent) or the wait runs out +% +% The proxy answers a rename or copy before it has completed it, so a +% test checks the result by polling the listing for up to a minute. + arguments + bucketName (1,1) string + objectName (1,1) string + options.Present (1,1) logical = true + options.TimeoutSeconds (1,1) double = 60 + end + tBegin = tic; + isSettled = false; + while ~isSettled && toc(tBegin) < options.TimeoutSeconds + client = ebrains.bucket.api.BucketsClient(); + isPresent = ebrains.bucket.internal.hasObject(client, bucketName, objectName); + isSettled = isPresent == options.Present; + if ~isSettled + pause(2) + end + end +end + function deleteIfPresent(bucketName, objectName) % deleteIfPresent - Best-effort cleanup of a probe object try