%PDF-
%PDF-
Mini Shell
Mini Shell
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\DAV\Upload;
use Exception;
use InvalidArgumentException;
use OC\Files\Filesystem;
use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\View;
use OC\Memcache\Memcached;
use OC\Memcache\Redis;
use OC_Hook;
use OCA\DAV\Connector\Sabre\Directory;
use OCA\DAV\Connector\Sabre\File;
use OCP\Files\IMimeTypeDetector;
use OCP\Files\IRootFolder;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\StorageInvalidException;
use OCP\ICache;
use OCP\ICacheFactory;
use OCP\IConfig;
use OCP\Lock\ILockingProvider;
use Sabre\DAV\Exception\BadRequest;
use Sabre\DAV\Exception\InsufficientStorage;
use Sabre\DAV\Exception\NotFound;
use Sabre\DAV\Exception\PreconditionFailed;
use Sabre\DAV\ICollection;
use Sabre\DAV\INode;
use Sabre\DAV\Server;
use Sabre\DAV\ServerPlugin;
use Sabre\HTTP\RequestInterface;
use Sabre\HTTP\ResponseInterface;
use Sabre\Uri;
class ChunkingV2Plugin extends ServerPlugin {
/** @var Server */
private $server;
/** @var UploadFolder */
private $uploadFolder;
/** @var ICache */
private $cache;
private ?string $uploadId = null;
private ?string $uploadPath = null;
private const TEMP_TARGET = '.target';
public const CACHE_KEY = 'chunking-v2';
public const UPLOAD_TARGET_PATH = 'upload-target-path';
public const UPLOAD_TARGET_ID = 'upload-target-id';
public const UPLOAD_ID = 'upload-id';
private const DESTINATION_HEADER = 'Destination';
public function __construct(ICacheFactory $cacheFactory) {
$this->cache = $cacheFactory->createDistributed(self::CACHE_KEY);
}
/**
* @inheritdoc
*/
public function initialize(Server $server) {
$server->on('afterMethod:MKCOL', [$this, 'afterMkcol']);
$server->on('beforeMethod:PUT', [$this, 'beforePut']);
$server->on('beforeMethod:DELETE', [$this, 'beforeDelete']);
$server->on('beforeMove', [$this, 'beforeMove'], 90);
$this->server = $server;
}
/**
* @param string $path
* @param bool $createIfNotExists
* @return FutureFile|UploadFile|ICollection|INode
*/
private function getUploadFile(string $path, bool $createIfNotExists = false) {
try {
$actualFile = $this->server->tree->getNodeForPath($path);
// Only directly upload to the target file if it is on the same storage
// There may be further potential to optimize here by also uploading
// to other storages directly. This would require to also carefully pick
// the storage/path used in getStorage()
if ($actualFile instanceof File && $this->uploadFolder->getStorage()->getId() === $actualFile->getNode()->getStorage()->getId()) {
return $actualFile;
}
} catch (NotFound $e) {
// If there is no target file we upload to the upload folder first
}
// Use file in the upload directory that will be copied or moved afterwards
if ($createIfNotExists) {
$this->uploadFolder->createFile(self::TEMP_TARGET);
}
/** @var UploadFile $uploadFile */
$uploadFile = $this->uploadFolder->getChild(self::TEMP_TARGET);
return $uploadFile->getFile();
}
public function afterMkcol(RequestInterface $request, ResponseInterface $response): bool {
try {
$this->prepareUpload($request->getPath());
$this->checkPrerequisites(false);
} catch (BadRequest|StorageInvalidException|NotFound $e) {
return true;
}
$this->uploadPath = $this->server->calculateUri($this->server->httpRequest->getHeader(self::DESTINATION_HEADER));
$targetFile = $this->getUploadFile($this->uploadPath, true);
[$storage, $storagePath] = $this->getUploadStorage($this->uploadPath);
$this->uploadId = $storage->startChunkedWrite($storagePath);
$this->cache->set($this->uploadFolder->getName(), [
self::UPLOAD_ID => $this->uploadId,
self::UPLOAD_TARGET_PATH => $this->uploadPath,
self::UPLOAD_TARGET_ID => $targetFile->getId(),
], 86400);
$response->setStatus(201);
return true;
}
public function beforePut(RequestInterface $request, ResponseInterface $response): bool {
try {
$this->prepareUpload(dirname($request->getPath()));
$this->checkPrerequisites();
} catch (StorageInvalidException|BadRequest|NotFound $e) {
return true;
}
[$storage, $storagePath] = $this->getUploadStorage($this->uploadPath);
$chunkName = basename($request->getPath());
$partId = is_numeric($chunkName) ? (int)$chunkName : -1;
if (!($partId >= 1 && $partId <= 10000)) {
throw new BadRequest('Invalid chunk name, must be numeric between 1 and 10000');
}
$uploadFile = $this->getUploadFile($this->uploadPath);
$tempTargetFile = null;
$additionalSize = (int)$request->getHeader('Content-Length');
if ($this->uploadFolder->childExists(self::TEMP_TARGET) && $this->uploadPath) {
/** @var UploadFile $tempTargetFile */
$tempTargetFile = $this->uploadFolder->getChild(self::TEMP_TARGET);
[$destinationDir, $destinationName] = Uri\split($this->uploadPath);
/** @var Directory $destinationParent */
$destinationParent = $this->server->tree->getNodeForPath($destinationDir);
$free = $destinationParent->getNode()->getFreeSpace();
$newSize = $tempTargetFile->getSize() + $additionalSize;
if ($free >= 0 && ($tempTargetFile->getSize() > $free || $newSize > $free)) {
throw new InsufficientStorage("Insufficient space in $this->uploadPath");
}
}
$stream = $request->getBodyAsStream();
$storage->putChunkedWritePart($storagePath, $this->uploadId, (string)$partId, $stream, $additionalSize);
$storage->getCache()->update($uploadFile->getId(), ['size' => $uploadFile->getSize() + $additionalSize]);
if ($tempTargetFile) {
$storage->getPropagator()->propagateChange($tempTargetFile->getInternalPath(), time(), $additionalSize);
}
$response->setStatus(201);
return false;
}
public function beforeMove($sourcePath, $destination): bool {
try {
$this->prepareUpload(dirname($sourcePath));
$this->checkPrerequisites();
} catch (StorageInvalidException|BadRequest|NotFound|PreconditionFailed $e) {
return true;
}
[$storage, $storagePath] = $this->getUploadStorage($this->uploadPath);
$targetFile = $this->getUploadFile($this->uploadPath);
[$destinationDir, $destinationName] = Uri\split($destination);
/** @var Directory $destinationParent */
$destinationParent = $this->server->tree->getNodeForPath($destinationDir);
$destinationExists = $destinationParent->childExists($destinationName);
// allow sync clients to send the modification and creation time along in a header
$updateFileInfo = [];
if ($this->server->httpRequest->getHeader('X-OC-MTime') !== null) {
$updateFileInfo['mtime'] = $this->sanitizeMtime($this->server->httpRequest->getHeader('X-OC-MTime'));
$this->server->httpResponse->setHeader('X-OC-MTime', 'accepted');
}
if ($this->server->httpRequest->getHeader('X-OC-CTime') !== null) {
$updateFileInfo['creation_time'] = $this->sanitizeMtime($this->server->httpRequest->getHeader('X-OC-CTime'));
$this->server->httpResponse->setHeader('X-OC-CTime', 'accepted');
}
$updateFileInfo['mimetype'] = \OCP\Server::get(IMimeTypeDetector::class)->detectPath($destinationName);
if ($storage->instanceOfStorage(ObjectStoreStorage::class) && $storage->getObjectStore() instanceof IObjectStoreMultiPartUpload) {
/** @var ObjectStoreStorage $storage */
/** @var IObjectStoreMultiPartUpload $objectStore */
$objectStore = $storage->getObjectStore();
$parts = $objectStore->getMultipartUploads($storage->getURN($targetFile->getId()), $this->uploadId);
$size = 0;
foreach ($parts as $part) {
$size += $part['Size'];
}
$free = $destinationParent->getNode()->getFreeSpace();
if ($free >= 0 && ($size > $free)) {
throw new InsufficientStorage("Insufficient space in $this->uploadPath");
}
}
$destinationInView = $destinationParent->getFileInfo()->getPath() . '/' . $destinationName;
$this->completeChunkedWrite($destinationInView);
$rootView = new View();
$rootView->putFileInfo($destinationInView, $updateFileInfo);
$sourceNode = $this->server->tree->getNodeForPath($sourcePath);
if ($sourceNode instanceof FutureFile) {
$this->uploadFolder->delete();
}
$this->server->emit('afterMove', [$sourcePath, $destination]);
$this->server->emit('afterUnbind', [$sourcePath]);
$this->server->emit('afterBind', [$destination]);
$response = $this->server->httpResponse;
$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
$response->setHeader('Content-Length', '0');
$response->setStatus($destinationExists ? 204 : 201);
return false;
}
public function beforeDelete(RequestInterface $request, ResponseInterface $response) {
try {
$this->prepareUpload(dirname($request->getPath()));
$this->checkPrerequisites();
} catch (StorageInvalidException|BadRequest|NotFound $e) {
return true;
}
[$storage, $storagePath] = $this->getUploadStorage($this->uploadPath);
$storage->cancelChunkedWrite($storagePath, $this->uploadId);
return true;
}
/**
* @throws BadRequest
* @throws PreconditionFailed
* @throws StorageInvalidException
*/
private function checkPrerequisites(bool $checkUploadMetadata = true): void {
$distributedCacheConfig = \OCP\Server::get(IConfig::class)->getSystemValue('memcache.distributed', null);
if ($distributedCacheConfig === null || (!$this->cache instanceof Redis && !$this->cache instanceof Memcached)) {
throw new BadRequest('Skipping chunking v2 since no proper distributed cache is available');
}
if (!$this->uploadFolder instanceof UploadFolder || empty($this->server->httpRequest->getHeader(self::DESTINATION_HEADER))) {
throw new BadRequest('Skipping chunked file writing as the destination header was not passed');
}
if (!$this->uploadFolder->getStorage()->instanceOfStorage(IChunkedFileWrite::class)) {
throw new StorageInvalidException('Storage does not support chunked file writing');
}
if ($this->uploadFolder->getStorage()->instanceOfStorage(ObjectStoreStorage::class) && !$this->uploadFolder->getStorage()->getObjectStore() instanceof IObjectStoreMultiPartUpload) {
throw new StorageInvalidException('Storage does not support multi part uploads');
}
if ($checkUploadMetadata) {
if ($this->uploadId === null || $this->uploadPath === null) {
throw new PreconditionFailed('Missing metadata for chunked upload. The distributed cache does not hold the information of previous requests.');
}
}
}
/**
* @return array [IStorage, string]
*/
private function getUploadStorage(string $targetPath): array {
$storage = $this->uploadFolder->getStorage();
$targetFile = $this->getUploadFile($targetPath);
return [$storage, $targetFile->getInternalPath()];
}
protected function sanitizeMtime(string $mtimeFromRequest): int {
if (!is_numeric($mtimeFromRequest)) {
throw new InvalidArgumentException('X-OC-MTime header must be an integer (unix timestamp).');
}
return (int)$mtimeFromRequest;
}
/**
* @throws NotFound
*/
public function prepareUpload($path): void {
$this->uploadFolder = $this->server->tree->getNodeForPath($path);
$uploadMetadata = $this->cache->get($this->uploadFolder->getName());
$this->uploadId = $uploadMetadata[self::UPLOAD_ID] ?? null;
$this->uploadPath = $uploadMetadata[self::UPLOAD_TARGET_PATH] ?? null;
}
private function completeChunkedWrite(string $targetAbsolutePath): void {
$uploadFile = $this->getUploadFile($this->uploadPath)->getNode();
[$storage, $storagePath] = $this->getUploadStorage($this->uploadPath);
$rootFolder = \OCP\Server::get(IRootFolder::class);
$exists = $rootFolder->nodeExists($targetAbsolutePath);
$uploadFile->lock(ILockingProvider::LOCK_SHARED);
$this->emitPreHooks($targetAbsolutePath, $exists);
try {
$uploadFile->changeLock(ILockingProvider::LOCK_EXCLUSIVE);
$storage->completeChunkedWrite($storagePath, $this->uploadId);
$uploadFile->changeLock(ILockingProvider::LOCK_SHARED);
} catch (Exception $e) {
$uploadFile->unlock(ILockingProvider::LOCK_EXCLUSIVE);
throw $e;
}
// If the file was not uploaded to the user storage directly we need to copy/move it
try {
$uploadFileAbsolutePath = $uploadFile->getFileInfo()->getPath();
if ($uploadFileAbsolutePath !== $targetAbsolutePath) {
$uploadFile = $rootFolder->get($uploadFile->getFileInfo()->getPath());
if ($exists) {
$uploadFile->copy($targetAbsolutePath);
} else {
$uploadFile->move($targetAbsolutePath);
}
}
$this->emitPostHooks($targetAbsolutePath, $exists);
} catch (Exception $e) {
$uploadFile->unlock(ILockingProvider::LOCK_SHARED);
throw $e;
}
}
private function emitPreHooks(string $target, bool $exists): void {
$hookPath = $this->getHookPath($target);
if (!$exists) {
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_create, [
Filesystem::signal_param_path => $hookPath,
]);
} else {
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_update, [
Filesystem::signal_param_path => $hookPath,
]);
}
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_write, [
Filesystem::signal_param_path => $hookPath,
]);
}
private function emitPostHooks(string $target, bool $exists): void {
$hookPath = $this->getHookPath($target);
if (!$exists) {
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_create, [
Filesystem::signal_param_path => $hookPath,
]);
} else {
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_update, [
Filesystem::signal_param_path => $hookPath,
]);
}
OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_write, [
Filesystem::signal_param_path => $hookPath,
]);
}
private function getHookPath(string $path): ?string {
if (!Filesystem::getView()) {
return $path;
}
return Filesystem::getView()->getRelativePath($path);
}
}
Zerion Mini Shell 1.0