Chunked file upload

src/Database/Factories/StorageRequestFactory.php

@@ -24,7 +24,6 @@ public function definition()
     {
         return [
             'user_id' => User::factory(),
-            'files' => [],
             'expires_at' => null,
             'submitted_at' => null,
         ];

src/Database/Factories/StorageRequestFileFactory.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace Biigle\Modules\UserStorage\Database\Factories;
+
+use Biigle\Modules\UserStorage\StorageRequest;
+use Biigle\Modules\UserStorage\StorageRequestFile;
+use Illuminate\Database\Eloquent\Factories\Factory;
+
+class StorageRequestFileFactory extends Factory
+{
+    /**
+     * The name of the factory's corresponding model.
+     *
+     * @var string
+     */
+    protected $model = StorageRequestFile::class;
+
+    /**
+     * Define the model's default state.
+     *
+     * @return array
+     */
+    public function definition()
+    {
+        return [
+            'path' => 'my/file.jpg',
+            'storage_request_id' => StorageRequest::factory(),
+            'size' => 123,
+            'received_chunks' => null,
+            'total_chunks' => null,
+        ];
+    }
+}

src/Http/Controllers/Api/StorageRequestController.php

@@ -9,10 +9,12 @@
 use Biigle\Modules\UserStorage\Http\Requests\StoreStorageRequest;
 use Biigle\Modules\UserStorage\Http\Requests\UpdateStorageRequest;
 use Biigle\Modules\UserStorage\Jobs\ApproveStorageRequest as ApproveStorageRequestJob;
+use Biigle\Modules\UserStorage\Jobs\AssembleChunkedFile;
 use Biigle\Modules\UserStorage\Notifications\StorageRequestRejected;
 use Biigle\Modules\UserStorage\Notifications\StorageRequestSubmitted;
 use Biigle\Modules\UserStorage\StorageRequest;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Bus;
 use Notification;
 
 class StorageRequestController extends Controller
@@ -51,7 +53,7 @@ public function store(StoreStorageRequest $request)
      */
     public function show($id)
     {
-        $request = StorageRequest::findOrFail($id);
+        $request = StorageRequest::with('files')->findOrFail($id);
         $this->authorize('access', $request);
 
         return $request;
@@ -74,8 +76,18 @@ public function update(UpdateStorageRequest $request)
     {
         $storageRequest = $request->storageRequest;
         $storageRequest->update(['submitted_at' => now()]);
-        Notification::route('mail', config('biigle.admin_email'))
-            ->notify(new StorageRequestSubmitted($storageRequest));
+
+        if ($request->chunkedFiles->isNotEmpty()) {
+            Bus::batch($request->chunkedFiles->map(function ($file) {
+                return new AssembleChunkedFile($file);
+            }))->then(function () use ($storageRequest) {
+                Notification::route('mail', config('biigle.admin_email'))
+                    ->notify(new StorageRequestSubmitted($storageRequest));
+            })->dispatch();
+        } else {
+            Notification::route('mail', config('biigle.admin_email'))
+                ->notify(new StorageRequestSubmitted($storageRequest));
+        }
     }
 
     /**
@@ -137,7 +149,6 @@ public function extend(ExtendStorageRequest $request)
     {
         $months = config('user_storage.expires_months');
         $request->storageRequest->update(['expires_at' => now()->addMonths($months)]);
-        $request->storageRequest->setHidden(['files']);
 
         return $request->storageRequest;
     }

src/Http/Controllers/Api/StorageRequestDirectoryController.php

@@ -4,7 +4,9 @@
 
 use Biigle\Http\Controllers\Api\Controller;
 use Biigle\Modules\UserStorage\Http\Requests\DestroyStorageRequestDirectory;
-use Biigle\Modules\UserStorage\Jobs\DeleteStorageRequestFiles;
+use Biigle\Modules\UserStorage\Jobs\DeleteStorageRequestFile;
+use Biigle\Modules\UserStorage\StorageRequestFile;
+use Queue;
 
 class StorageRequestDirectoryController extends Controller
 {
@@ -25,11 +27,11 @@ class StorageRequestDirectoryController extends Controller
      */
     public function destroy(DestroyStorageRequestDirectory $request)
     {
-        DeleteStorageRequestFiles::dispatch($request->storageRequest, $request->files);
+        $request->files->load('request');
+        Queue::bulk($request->files->map(function ($file) {
+            return new DeleteStorageRequestFile($file);
+        }));
 
-        $request->storageRequest->files = array_values(array_diff(
-            $request->storageRequest->files, $request->files
-        ));
-        $request->storageRequest->save();
+        StorageRequestFile::whereIn('id', $request->files->pluck('id'))->delete();
     }
 }

src/Http/Controllers/Api/StorageRequestFileController.php

@@ -5,8 +5,9 @@
 use Biigle\Http\Controllers\Api\Controller;
 use Biigle\Modules\UserStorage\Http\Requests\DestroyStorageRequestFile;
 use Biigle\Modules\UserStorage\Http\Requests\StoreStorageRequestFile;
-use Biigle\Modules\UserStorage\Jobs\DeleteStorageRequestFiles;
+use Biigle\Modules\UserStorage\Jobs\DeleteStorageRequestFile;
 use Biigle\Modules\UserStorage\StorageRequest;
+use Biigle\Modules\UserStorage\StorageRequestFile;
 use Biigle\Modules\UserStorage\User;
 use DB;
 use Exception;
@@ -40,26 +41,55 @@ class StorageRequestFileController extends Controller
      * @apiParam (Required arguments) {File} file The file to add to the storage request.
      *
      * @apiParam (Optional arguments) {string} prefix Optional prefix to prepend to the filename. Use slashes to create directories.
+     * @apiParam (Optional arguments) {int} chunk_index Index of the uploaded chunk in case the file is uploaded in chunks. The first chunk must be uploaded first.
+     * @apiParam (Optional arguments) {int} chunk_total Total number of chunks for this file in case the file is uploaded in chunks.
      *
      * @param StoreStorageRequestFile $request
      *
      * @return \Illuminate\Http\Response
      */
     public function store(StoreStorageRequestFile $request)
     {
-        DB::transaction(function () use ($request) {
+        return DB::transaction(function () use ($request) {
             $sr = $request->storageRequest;
 
             $file = $request->file('file');
-            $filePath = $request->getFilePath();
             $disk = config('user_storage.pending_disk');
+            $filePath = $request->getFilePath();
+            $fileModel = $request->storageRequestFile;
+
+            if ($request->isChunked()) {
+                $chunkIndex = (int) $request->input('chunk_index');
 
-            $user = User::convert($sr->user);
-            $user->storage_quota_used += $file->getSize();
-            $user->save();
+                if ($fileModel) {
+                    $fileModel->update([
+                        'size' => $fileModel->size + $file->getSize(),
+                        'received_chunks' => array_merge($fileModel->received_chunks, [$chunkIndex]),
+                    ]);
+                } elseif ($chunkIndex === 0) {
+                    $fileModel = $sr->files()->create([
+                        'path' => $filePath,
+                        'size' => $file->getSize(),
+                        'received_chunks' => [0],
+                        'total_chunks' => $request->input('chunk_total'),
+                    ]);
+                } else {
+                    // This should never be allowed by the validation.
+                    throw new Exception('The first chunk must be uploaded first.');
+                }
 
-            $sr->files = array_merge($sr->files, [$filePath]);
-            $sr->save();
+                $filePath .= '.'.$chunkIndex;
+
+            } else {
+                if ($fileModel) {
+                    $fileModel->update(['size' => $file->getSize()]);
+                } else {
+                    $fileModel = $sr->files()->create([
+                        'path' => $filePath,
+                        'size' => $file->getSize(),
+                    ]);
+                }
+            }
 
             // Retry the upload a few times, as we observed storage backends that threw
             // random errors which did not happen again after a retry.
@@ -81,20 +111,20 @@ public function store(StoreStorageRequestFile $request)
             if ($success === false) {
                 throw new Exception("Unable to save file.");
             }
+
+            return $fileModel;
         });
     }
 
     /**
      * Show a file of a storage request.
      *
-     * @api {get} storage-requests/:id/files/:path Show a file
+     * @api {get} storage-request-files/:id Show a file
      * @apiGroup UserStorage
      * @apiName ShowStorageRequestFile
      * @apiPermission admin
      *
-     * @apiParam {Number} id The storage request ID
-     *
-     * @apiParam (Required parameters) {String} path The file path
+     * @apiParam {Number} id The storage request file ID
      *
      * @param Request $request
      * @param int $id
@@ -107,19 +137,14 @@ public function show(Request $request, $id)
             abort(Response::HTTP_NOT_FOUND);
         }
 
-        $storageRequest = StorageRequest::findOrFail($id);
-        $path = $request->input('path');
+        $file = StorageRequestFile::with('request')->findOrFail($id);
 
-        if (!in_array($path, $storageRequest->files)) {
-            abort(Response::HTTP_NOT_FOUND);
-        }
-
-        if (is_null($storageRequest->expires_at)) {
+        if (is_null($file->request->expires_at)) {
                 $disk = Storage::disk(config('user_storage.pending_disk'));
-                $path = $storageRequest->getPendingPath($path);
+                $path = $file->request->getPendingPath($file->path);
             } else {
                 $disk = Storage::disk(config('user_storage.storage_disk'));
-                $path = $storageRequest->getStoragePath($path);
+                $path = $file->request->getStoragePath($file->path);
             }
 
         try {
@@ -145,26 +170,19 @@ public function show(Request $request, $id)
     /**
      * Delete files of a storage request
      *
-     * @api {delete} storage-requests/:id/files Delete files
+     * @api {delete} storage-request-files/:id Delete files
      * @apiGroup UserStorage
      * @apiName DestroyStorageRequestFile
      * @apiPermission storageRequestOwner
      *
-     * @apiParam {Number} id The storage request ID.
-     *
-     * @apiParam (Required arguments) {File[]} files Array of file paths that should be deleted from this storage request.
+     * @apiParam {Number} id The storage request file ID.
      *
      * @param DestroyStorageRequestFile $request
      * @return \Illuminate\Http\Response
      */
     public function destroy(DestroyStorageRequestFile $request)
     {
-        $files = $request->input('files');
-        DeleteStorageRequestFiles::dispatch($request->storageRequest, $files);
-
-        $request->storageRequest->files = array_values(array_diff(
-            $request->storageRequest->files, $files
-        ));
-        $request->storageRequest->save();
+        DeleteStorageRequestFile::dispatch($request->file);
+        $request->file->delete();
     }
 }

src/Http/Controllers/Views/StorageRequestController.php

@@ -26,10 +26,7 @@ public function index(Request $request)
         $requests = StorageRequest::where('user_id', $user->id)
             ->whereNotNull('submitted_at')
             ->orderBy('submitted_at', 'desc')
-            ->get()
-            ->each(function ($request) {
-                $request->setHidden(['files']);
-            });
+            ->get();
 
         $expireDate = now()->addWeeks(config('user_storage.about_to_expire_weeks'));
 
@@ -55,8 +52,10 @@ public function create(Request $request)
         $usedQuota = $user->storage_quota_used;
         $availableQuota = $user->storage_quota_available;
         $maxFilesize = config('user_storage.max_file_size');
+        $chunkSize = config('user_storage.upload_chunk_size');
 
         $previousRequest = StorageRequest::whereNull('submitted_at')
+            ->with('files')
             ->where('user_id', $user->id)
             ->first();
 
@@ -66,6 +65,7 @@ public function create(Request $request)
             'usedQuota' => $usedQuota,
             'availableQuota' => $availableQuota,
             'maxFilesize' => $maxFilesize,
+            'chunkSize' => $chunkSize,
         ]);
     }
 
@@ -78,7 +78,9 @@ public function create(Request $request)
      */
     public function review($id)
     {
-        $request = StorageRequest::whereNull('expires_at')->findOrFail($id);
+        $request = StorageRequest::whereNull('expires_at')
+            ->with('files')
+            ->findOrFail($id);
         $this->authorize('approve', $request);
 
         return view('user-storage::review', [