Skip to content

Commit 5e286f6

Browse files
johannesboynejohannesboyne
authored
Merge pull request #107 from Diggs/master
 Conditional Write (IfMatch and IfNoneMatch) support for PutObject
2 parents ed9094b + 952b293 commit 5e286f6

File tree

12 files changed

+673
-39
lines changed

12 files changed

+673
-39
lines changed

‎backend.go‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,69 @@ type PutObjectResult struct {
127127
VersionID VersionID
128128
}
129129

130+
// PutConditions represents the conditional headers for S3 PutObject operations.
131+
// These conditions are checked atomically before writing the object.
132+
type PutConditions struct {
133+
// IfMatch specifies that the object should only be written if its ETag
134+
// matches this value. If the ETag doesn't match, ErrPreconditionFailed
135+
// should be returned.
136+
IfMatch *string
137+
138+
// IfNoneMatch specifies that the object should only be written if the
139+
// object key doesn't already exist. The value should be "*".
140+
// If the object exists, ErrPreconditionFailed should be returned.
141+
IfNoneMatch *string
142+
}
143+
144+
// ConditionalObjectInfo represents the current state of an object for conditional checking.
145+
// Backends should provide this information to the shared CheckPutConditions function.
146+
type ConditionalObjectInfo struct {
147+
// Exists indicates whether the object exists
148+
Exists bool
149+
150+
// Hash is the MD5 hash of the object content (used for ETag comparison)
151+
// Only required if Exists is true
152+
Hash []byte
153+
}
154+
155+
// ObjectInfo is a deprecated alias for ConditionalObjectInfo.
156+
// Use ConditionalObjectInfo instead.
157+
type ObjectInfo = ConditionalObjectInfo
158+
159+
// FormatETag formats a hash as an S3 ETag with quotes
160+
func FormatETag(hash []byte) string {
161+
return `"` + hex.EncodeToString(hash) + `"`
162+
}
163+
164+
// CheckPutConditions validates conditional headers for PutObject operations.
165+
// This is a shared implementation that all backends can use.
166+
func CheckPutConditions(conditions *PutConditions, objectInfo *ConditionalObjectInfo) error {
167+
// Check If-None-Match: object should not exist
168+
if conditions.IfNoneMatch != nil {
169+
if *conditions.IfNoneMatch == "*" && objectInfo.Exists {
170+
return ErrorMessage(ErrPreconditionFailed, "The object already exists")
171+
}
172+
}
173+
174+
// Check If-Match: if specified, object must exist and ETag must match
175+
if conditions.IfMatch != nil {
176+
if !objectInfo.Exists {
177+
return ErrorMessage(ErrPreconditionFailed, "The object does not exist")
178+
}
179+
expectedETag := *conditions.IfMatch
180+
// Remove quotes if present for comparison
181+
if len(expectedETag) >= 2 && expectedETag[0] == '"' && expectedETag[len(expectedETag)-1] == '"' {
182+
expectedETag = expectedETag[1 : len(expectedETag)-1]
183+
}
184+
actualETag := hex.EncodeToString(objectInfo.Hash)
185+
if expectedETag != actualETag {
186+
return ErrorMessage(ErrPreconditionFailed, "The ETag does not match")
187+
}
188+
}
189+
190+
return nil
191+
}
192+
130193
// Backend provides a set of operations to be implemented in order to support
131194
// gofakes3.
132195
//
@@ -234,7 +297,11 @@ type Backend interface {
234297
//
235298
// The size can be used if the backend needs to read the whole reader; use
236299
// gofakes3.ReadAll() for this job rather than ioutil.ReadAll().
237-
PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64) (PutObjectResult, error)
300+
//
301+
// If conditions is not nil, the backend should check the conditions before
302+
// writing the object. If conditions fail, it should return ErrPreconditionFailed
303+
// or ErrConditionalRequestConflict as appropriate.
304+
PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64, conditions *PutConditions) (PutObjectResult, error)
238305

239306
DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error)
240307

@@ -344,7 +411,7 @@ func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta ma
344411
}
345412
defer c.Contents.Close()
346413

347-
_, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size)
414+
_, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size, nil)
348415
if err != nil {
349416
return
350417
}

‎backend/s3afero/backend_test.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ import (
1010
"reflect"
1111
"testing"
1212

13-
"github.com/johannesboyne/gofakes3"
1413
"github.com/spf13/afero"
14+
15+
"github.com/johannesboyne/gofakes3"
1516
)
1617

1718
func testingBackends(t *testing.T) []gofakes3.Backend {
@@ -43,7 +44,7 @@ func TestPutGet(t *testing.T) {
4344
}
4445

4546
contents := []byte("contents")
46-
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
47+
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents)), nil); err != nil {
4748
t.Fatal(err)
4849
}
4950
hasher := md5.New()
@@ -89,7 +90,7 @@ func TestPutGetRange(t *testing.T) {
8990

9091
contents := []byte("contents")
9192
expected := contents[1:7]
92-
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
93+
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents)), nil); err != nil {
9394
t.Fatal(err)
9495
}
9596
hasher := md5.New()
@@ -134,12 +135,12 @@ func TestPutListRoot(t *testing.T) {
134135
}
135136

136137
contents1 := []byte("contents1")
137-
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
138+
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents1), int64(len(contents1)), nil); err != nil {
138139
t.Fatal(err)
139140
}
140141

141142
contents2 := []byte("contents2")
142-
if _, err := backend.PutObject("test", "bar", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
143+
if _, err := backend.PutObject("test", "bar", meta, bytes.NewReader(contents2), int64(len(contents2)), nil); err != nil {
143144
t.Fatal(err)
144145
}
145146

@@ -180,12 +181,12 @@ func TestPutListDir(t *testing.T) {
180181
}
181182

182183
contents1 := []byte("contents1")
183-
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
184+
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1)), nil); err != nil {
184185
t.Fatal(err)
185186
}
186187

187188
contents2 := []byte("contents2")
188-
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
189+
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2)), nil); err != nil {
189190
t.Fatal(err)
190191
}
191192

@@ -226,7 +227,7 @@ func TestPutDelete(t *testing.T) {
226227
}
227228

228229
contents := []byte("contents1")
229-
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
230+
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents), int64(len(contents)), nil); err != nil {
230231
t.Fatal(err)
231232
}
232233

@@ -258,12 +259,12 @@ func TestPutDeleteMulti(t *testing.T) {
258259
}
259260

260261
contents1 := []byte("contents1")
261-
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
262+
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1)), nil); err != nil {
262263
t.Fatal(err)
263264
}
264265

265266
contents2 := []byte("contents2")
266-
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
267+
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2)), nil); err != nil {
267268
t.Fatal(err)
268269
}
269270

‎backend/s3afero/multi.go‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package s3afero
22

33
import (
44
"crypto/md5"
5-
"encoding/hex"
65
"fmt"
76
"io"
87
"log"
@@ -12,9 +11,10 @@ import (
1211
"strings"
1312
"sync"
1413

14+
"github.com/spf13/afero"
15+
1516
"github.com/johannesboyne/gofakes3"
1617
"github.com/johannesboyne/gofakes3/internal/s3io"
17-
"github.com/spf13/afero"
1818
)
1919

2020
// MultiBucketBackend is a gofakes3.Backend that allows you to create multiple
@@ -165,7 +165,7 @@ func (db *MultiBucketBackend) getBucketWithFilePrefixLocked(bucket string, prefi
165165
response.Add(&gofakes3.Content{
166166
Key: objectPath,
167167
LastModified: gofakes3.NewContentTime(mtime),
168-
ETag: `"` + hex.EncodeToString(meta.Hash) + `"`,
168+
ETag: gofakes3.FormatETag(meta.Hash),
169169
Size: size,
170170
})
171171
}
@@ -212,7 +212,7 @@ func (db *MultiBucketBackend) getBucketWithArbitraryPrefixLocked(bucket string,
212212
response.Add(&gofakes3.Content{
213213
Key: objectName,
214214
LastModified: gofakes3.NewContentTime(mtime),
215-
ETag: `"` + hex.EncodeToString(meta.Hash) + `"`,
215+
ETag: gofakes3.FormatETag(meta.Hash),
216216
Size: size,
217217
})
218218

@@ -419,7 +419,9 @@ func (db *MultiBucketBackend) GetObject(bucketName, objectName string, rangeRequ
419419
func (db *MultiBucketBackend) PutObject(
420420
bucketName, objectName string,
421421
meta map[string]string,
422-
input io.Reader, size int64,
422+
input io.Reader,
423+
size int64,
424+
conditions *gofakes3.PutConditions,
423425
) (result gofakes3.PutObjectResult, err error) {
424426

425427
err = gofakes3.MergeMetadata(db, bucketName, objectName, meta)
@@ -438,6 +440,16 @@ func (db *MultiBucketBackend) PutObject(
438440
return result, gofakes3.BucketNotFound(bucketName)
439441
}
440442

443+
if conditions != nil {
444+
objectInfo, err := db.getConditionalObjectInfo(bucketName, objectName)
445+
if err != nil {
446+
return result, err
447+
}
448+
if err := gofakes3.CheckPutConditions(conditions, objectInfo); err != nil {
449+
return result, err
450+
}
451+
}
452+
441453
objectPath := path.Join(bucketName, objectName)
442454
objectFilePath := filepath.FromSlash(objectPath)
443455
objectDir := filepath.Dir(objectFilePath)
@@ -558,3 +570,29 @@ func (db *MultiBucketBackend) DeleteMulti(bucketName string, objects ...string)
558570

559571
return result, nil
560572
}
573+
574+
// getConditionalObjectInfo returns information about an object for conditional checking.
575+
// This method assumes the backend lock is already held.
576+
func (db *MultiBucketBackend) getConditionalObjectInfo(bucketName, objectName string) (*gofakes3.ConditionalObjectInfo, error) {
577+
fullPath := path.Join(bucketName, objectName)
578+
579+
stat, err := db.bucketFs.Stat(filepath.FromSlash(fullPath))
580+
if os.IsNotExist(err) {
581+
return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
582+
} else if err != nil {
583+
return nil, err
584+
} else if stat.IsDir() {
585+
return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
586+
}
587+
588+
size, mtime := stat.Size(), stat.ModTime()
589+
meta, err := db.metaStore.loadMeta(bucketName, objectName, size, mtime)
590+
if err != nil {
591+
return nil, err
592+
}
593+
594+
return &gofakes3.ConditionalObjectInfo{
595+
Exists: true,
596+
Hash: meta.Hash,
597+
}, nil
598+
}

‎backend/s3afero/single.go‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package s3afero
22

33
import (
44
"crypto/md5"
5-
"encoding/hex"
65
"errors"
76
"io"
87
"log"
@@ -13,9 +12,10 @@ import (
1312
"sync"
1413
"time"
1514

15+
"github.com/spf13/afero"
16+
1617
"github.com/johannesboyne/gofakes3"
1718
"github.com/johannesboyne/gofakes3/internal/s3io"
18-
"github.com/spf13/afero"
1919
)
2020

2121
// SingleBucketBackend is a gofakes3.Backend that allows you to treat an existing
@@ -144,7 +144,7 @@ func (db *SingleBucketBackend) getBucketWithFilePrefixLocked(bucket string, pref
144144
response.Add(&gofakes3.Content{
145145
Key: objectPath,
146146
LastModified: gofakes3.NewContentTime(mtime),
147-
ETag: `"` + hex.EncodeToString(meta.Hash) + `"`,
147+
ETag: gofakes3.FormatETag(meta.Hash),
148148
Size: size,
149149
})
150150
}
@@ -176,7 +176,7 @@ func (db *SingleBucketBackend) getBucketWithArbitraryPrefixLocked(bucket string,
176176
response.Add(&gofakes3.Content{
177177
Key: objectPath,
178178
LastModified: gofakes3.NewContentTime(mtime),
179-
ETag: `"` + hex.EncodeToString(meta.Hash) + `"`,
179+
ETag: gofakes3.FormatETag(meta.Hash),
180180
Size: size,
181181
})
182182

@@ -322,7 +322,9 @@ func (db *SingleBucketBackend) GetObject(bucketName, objectName string, rangeReq
322322
func (db *SingleBucketBackend) PutObject(
323323
bucketName, objectName string,
324324
meta map[string]string,
325-
input io.Reader, size int64,
325+
input io.Reader,
326+
size int64,
327+
conditions *gofakes3.PutConditions,
326328
) (result gofakes3.PutObjectResult, err error) {
327329

328330
if bucketName != db.name {
@@ -337,6 +339,16 @@ func (db *SingleBucketBackend) PutObject(
337339
db.lock.Lock()
338340
defer db.lock.Unlock()
339341

342+
if conditions != nil {
343+
objectInfo, err := db.getConditionalObjectInfo(bucketName, objectName)
344+
if err != nil {
345+
return result, err
346+
}
347+
if err := gofakes3.CheckPutConditions(conditions, objectInfo); err != nil {
348+
return result, err
349+
}
350+
}
351+
340352
objectFilePath := filepath.FromSlash(objectName)
341353
objectDir := filepath.Dir(objectFilePath)
342354

@@ -499,3 +511,27 @@ func (db *SingleBucketBackend) ForceDeleteBucket(name string) error {
499511
func (db *SingleBucketBackend) BucketExists(name string) (exists bool, err error) {
500512
return db.name == name, nil
501513
}
514+
515+
// getConditionalObjectInfo returns information about an object for conditional checking.
516+
// This method assumes the backend lock is already held.
517+
func (db *SingleBucketBackend) getConditionalObjectInfo(bucketName, objectName string) (*gofakes3.ConditionalObjectInfo, error) {
518+
stat, err := db.fs.Stat(filepath.FromSlash(objectName))
519+
if os.IsNotExist(err) {
520+
return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
521+
} else if err != nil {
522+
return nil, err
523+
} else if stat.IsDir() {
524+
return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
525+
}
526+
527+
size, mtime := stat.Size(), stat.ModTime()
528+
meta, err := db.ensureMeta(bucketName, objectName, size, mtime)
529+
if err != nil {
530+
return nil, err
531+
}
532+
533+
return &gofakes3.ConditionalObjectInfo{
534+
Exists: true,
535+
Hash: meta.Hash,
536+
}, nil
537+
}

0 commit comments

Comments
 (0)