Skip to content

Commit 09235cc

Browse files
Carl SchwanCarlSchwan
Carl Schwan
authored and
CarlSchwan
committed
perf(s3): Expose pre-signed urls for S3
This is faster than going back to nextcloud to download the files. This is an opt-in setting that can be enabled by setting use_presigned_url in the object store config. Additionally add support for the proxy config which is needed in a docker setup. See juliusknorr/nextcloud-docker-dev#431 Signed-off-by: Carl Schwan <[email protected]>
1 parent df8d838 commit 09235cc

File tree

12 files changed

+117
-24
lines changed

12 files changed

+117
-24
lines changed

‎apps/dav/lib/Connector/Sabre/File.php‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -538,19 +538,16 @@ public function getContentType() {
538538
return Server::get(IMimeTypeDetector::class)->getSecureMimeType($mimeType);
539539
}
540540

541-
/**
542-
* @return array|bool
543-
*/
544-
public function getDirectDownload() {
541+
public function getDirectDownload(): array|false {
545542
if (Server::get(IAppManager::class)->isEnabledForUser('encryption')) {
546-
return [];
543+
return false;
547544
}
548-
[$storage, $internalPath] = $this->fileView->resolvePath($this->path);
549-
if (is_null($storage)) {
550-
return [];
545+
$node = $this->getNode();
546+
$storage = $node->getStorage();
547+
if (!$storage) {
548+
return false;
551549
}
552-
553-
return $storage->getDirectDownload($internalPath);
550+
return $storage->getDirectDownloadById((string)$node->getId());
554551
}
555552

556553
/**

‎apps/dav/lib/Connector/Sabre/FilesPlugin.php‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class FilesPlugin extends ServerPlugin {
5050
public const OCM_SHARE_PERMISSIONS_PROPERTYNAME = '{http://open-cloud-mesh.org/ns}share-permissions';
5151
public const SHARE_ATTRIBUTES_PROPERTYNAME = '{http://nextcloud.org/ns}share-attributes';
5252
public const DOWNLOADURL_PROPERTYNAME = '{http://owncloud.org/ns}downloadURL';
53+
public const DOWNLOADURL_EXPIRATION_PROPERTYNAME = '{http://nextcloud.org/ns}download-url-expiration';
5354
public const SIZE_PROPERTYNAME = '{http://owncloud.org/ns}size';
5455
public const GETETAG_PROPERTYNAME = '{DAV:}getetag';
5556
public const LASTMODIFIED_PROPERTYNAME = '{DAV:}lastmodified';
@@ -120,6 +121,7 @@ public function initialize(Server $server) {
120121
$server->protectedProperties[] = self::SHARE_ATTRIBUTES_PROPERTYNAME;
121122
$server->protectedProperties[] = self::SIZE_PROPERTYNAME;
122123
$server->protectedProperties[] = self::DOWNLOADURL_PROPERTYNAME;
124+
$server->protectedProperties[] = self::DOWNLOADURL_EXPIRATION_PROPERTYNAME;
123125
$server->protectedProperties[] = self::OWNER_ID_PROPERTYNAME;
124126
$server->protectedProperties[] = self::OWNER_DISPLAY_NAME_PROPERTYNAME;
125127
$server->protectedProperties[] = self::CHECKSUMS_PROPERTYNAME;
@@ -474,14 +476,20 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
474476
$propFind->handle(self::DOWNLOADURL_PROPERTYNAME, function () use ($node) {
475477
try {
476478
$directDownloadUrl = $node->getDirectDownload();
477-
if (isset($directDownloadUrl['url'])) {
479+
if ($directDownloadUrl && isset($directDownloadUrl['url'])) {
478480
return $directDownloadUrl['url'];
479481
}
480-
} catch (StorageNotAvailableException $e) {
481-
return false;
482-
} catch (ForbiddenException $e) {
483-
return false;
484-
}
482+
} catch (StorageNotAvailableException|ForbiddenException) {}
483+
return false;
484+
});
485+
486+
$propFind->handle(self::DOWNLOADURL_EXPIRATION_PROPERTYNAME, function () use ($node) {
487+
try {
488+
$directDownloadUrl = $node->getDirectDownload();
489+
if ($directDownloadUrl && isset($directDownloadUrl['expiration'])) {
490+
return $directDownloadUrl['expiration'];
491+
}
492+
} catch (StorageNotAvailableException|ForbiddenException) {}
485493
return false;
486494
});
487495

‎lib/private/Files/ObjectStore/Azure.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,4 +117,8 @@ public function objectExists($urn) {
117117
public function copyObject($from, $to) {
118118
$this->getBlobClient()->copyBlob($this->containerName, $to, $this->containerName, $from);
119119
}
120+
121+
public function preSignedUrl(string $urn, \DateTimeInterface $expiration): ?string {
122+
return null;
123+
}
120124
}

‎lib/private/Files/ObjectStore/ObjectStoreStorage.php‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
use OCP\Files\Storage\IStorage;
3030
use OCP\IDBConnection;
3131
use OCP\Server;
32+
use Override;
3233
use Psr\Log\LoggerInterface;
3334

3435
class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
@@ -844,4 +845,22 @@ public function free_space(string $path): int|float|false {
844845

845846
return $available;
846847
}
848+
849+
#[Override]
850+
public function getDirectDownloadById(string $fileId): array|false {
851+
$expiration = new \DateTimeImmutable('+60 minutes');
852+
$url = $this->objectStore->preSignedUrl($this->getURN($fileId), $expiration);
853+
return $url ? ['url' => $url, 'expiration' => $expiration->getTimestamp()] : false;
854+
}
855+
856+
#[Override]
857+
public function getDirectDownload(string $path): array|false {
858+
$path = $this->normalizePath($path);
859+
$cacheEntry = $this->getCache()->get($path);
860+
861+
if (!$cacheEntry || $cacheEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
862+
return false;
863+
}
864+
$this->getDirectDownloadById($cacheEntry->getId());
865+
}
847866
}

‎lib/private/Files/ObjectStore/S3ConnectionTrait.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ trait S3ConnectionTrait {
2828

2929
protected ?S3Client $connection = null;
3030

31+
private bool $usePresignedUrl = false;
32+
3133
protected function parseParams($params) {
3234
if (empty($params['bucket'])) {
3335
throw new \Exception('Bucket has to be configured.');
@@ -93,12 +95,15 @@ public function getConnection() {
9395
)
9496
);
9597

98+
$this->usePresignedUrl = $this->params['use_presigned_url'] ?? false;
99+
96100
$options = [
97101
'version' => $this->params['version'] ?? 'latest',
98102
'credentials' => $provider,
99103
'endpoint' => $base_url,
100104
'region' => $this->params['region'],
101105
'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
106+
'proxy' => isset($this->params['proxy']) ? $this->params['proxy'] : false,
102107
'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()),
103108
'csm' => false,
104109
'use_arn_region' => false,
@@ -256,4 +261,8 @@ protected function getSSECParameters(bool $copy = false): array {
256261
'SSECustomerKeyMD5' => md5($rawKey, true)
257262
];
258263
}
264+
265+
public function isUsePresignedUrl(): bool {
266+
return $this->usePresignedUrl;
267+
}
259268
}

‎lib/private/Files/ObjectStore/S3ObjectTrait.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
namespace OC\Files\ObjectStore;
88

99
use Aws\Command;
10+
use Aws\Exception\AwsException;
1011
use Aws\Exception\MultipartUploadException;
1112
use Aws\S3\Exception\S3MultipartUploadException;
1213
use Aws\S3\MultipartCopy;
@@ -291,4 +292,23 @@ public function copyObject($from, $to, array $options = []) {
291292
], $options));
292293
}
293294
}
295+
296+
public function preSignedUrl(string $urn, \DateTimeInterface $expiration): ?string {
297+
$command = $this->getConnection()->getCommand('GetObject', [
298+
'Bucket' => $this->getBucket(),
299+
'Key' => $urn,
300+
]);
301+
302+
if (!$this->isUsePresignedUrl()) {
303+
return null;
304+
}
305+
306+
try {
307+
return (string)$this->getConnection()->createPresignedRequest($command, $expiration, [
308+
'signPayload' => true,
309+
])->getUri();
310+
} catch (AwsException) {
311+
return null;
312+
}
313+
}
294314
}

‎lib/private/Files/ObjectStore/StorageObjectStore.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,8 @@ public function objectExists($urn) {
7474
public function copyObject($from, $to) {
7575
$this->storage->copy($from, $to);
7676
}
77+
78+
public function preSignedUrl(string $urn, \DateTimeInterface $expiration): ?string {
79+
return null;
80+
}
7781
}

‎lib/private/Files/ObjectStore/Swift.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,7 @@ public function copyObject($from, $to) {
134134
'destination' => $this->getContainer()->name . '/' . $to
135135
]);
136136
}
137+
public function preSignedUrl(string $urn, \DateTimeInterface $expiration): ?string {
138+
return null;
139+
}
137140
}

‎lib/private/Files/Storage/Common.php‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
use OCP\Files\Cache\IScanner;
2626
use OCP\Files\Cache\IUpdater;
2727
use OCP\Files\Cache\IWatcher;
28+
use OCP\Files\FileInfo;
2829
use OCP\Files\ForbiddenException;
2930
use OCP\Files\GenericFileException;
3031
use OCP\Files\IFilenameValidator;
@@ -38,6 +39,7 @@
3839
use OCP\Lock\ILockingProvider;
3940
use OCP\Lock\LockedException;
4041
use OCP\Server;
42+
use Override;
4143
use Psr\Log\LoggerInterface;
4244

4345
/**
@@ -445,15 +447,16 @@ public function instanceOfStorage(string $class): bool {
445447
return is_a($this, $class);
446448
}
447449

448-
/**
449-
* A custom storage implementation can return an url for direct download of a give file.
450-
*
451-
* For now the returned array can hold the parameter url - in future more attributes might follow.
452-
*/
450+
#[Override]
453451
public function getDirectDownload(string $path): array|false {
454452
return [];
455453
}
456454

455+
#[Override]
456+
public function getDirectDownloadById(string $fileId): array|false {
457+
return [];
458+
}
459+
457460
public function verifyPath(string $path, string $fileName): void {
458461
$this->getFilenameValidator()
459462
->validateFilename($fileName);

‎lib/private/Files/Storage/Wrapper/Wrapper.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
use OCP\Files\Storage\IWriteStreamStorage;
2121
use OCP\Lock\ILockingProvider;
2222
use OCP\Server;
23+
use Override;
2324
use Psr\Log\LoggerInterface;
2425

2526
class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStreamStorage {
@@ -258,10 +259,16 @@ public function __call(string $method, array $args) {
258259
return call_user_func_array([$this->getWrapperStorage(), $method], $args);
259260
}
260261

262+
#[Override]
261263
public function getDirectDownload(string $path): array|false {
262264
return $this->getWrapperStorage()->getDirectDownload($path);
263265
}
264266

267+
#[Override]
268+
public function getDirectDownloadById(string $fileId): array|false {
269+
return $this->getWrapperStorage()->getDirectDownloadById($fileId);
270+
}
271+
265272
public function getAvailability(): array {
266273
return $this->getWrapperStorage()->getAvailability();
267274
}

0 commit comments

Comments
 (0)