From f39195a2efb0d9bea9288332daa5a8a4d3737800 Mon Sep 17 00:00:00 2001 From: Spitfire Date: Wed, 25 Oct 2023 13:27:02 -0600 Subject: [PATCH 1/5] wip --- ...5_173253_create_campaign_exports_table.php | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 database/migrations/2023_10_25_173253_create_campaign_exports_table.php diff --git a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php new file mode 100644 index 0000000000..0b08aefce9 --- /dev/null +++ b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php @@ -0,0 +1,71 @@ +id(); + $table->unsignedInteger('campaign_id'); + + + $table->timestamps(); + $table->unsignedInteger('created_by')->nullable(); + $table->unsignedInteger('updated_by')->nullable(); + + $table->foreign('created_by')->references('id')->on('users')->onDelete('set null'); + $table->foreign('updated_by')->references('id')->on('users')->onDelete('set null'); + + $table->mediumText('config')->nullable(); + + $table->unsignedInteger('family_id'); + $table->foreign('family_id')->references('id')->on('families')->cascadeOnDelete(); + + + + $table->bigIncrements('id'); + $table->unsignedInteger('campaign_id'); + $table->unsignedInteger('created_by')->nullable(); + + $table->string('name', 100); + + $table->index('campaign_id'); + $table->timestamps(); + + $table->foreign('campaign_id')->references('id')->on('campaigns')->onDelete('cascade'); + $table->foreign('created_by')->references('id')->on('users')->onDelete('set null'); + + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('campaign_exports'); + + } +}; From 208a64a09d7aed04b11eb95759e9e74fca775c87 Mon Sep 17 00:00:00 2001 From: Spitfire Date: Fri, 27 Oct 2023 10:48:54 -0600 Subject: [PATCH 2/5] Export Logs --- app/Jobs/Campaigns/Export.php | 22 +++++++-- app/Models/CampaignExport.php | 48 +++++++++++++++++++ app/Services/Campaign/ExportService.php | 27 +++++++++-- ...5_173253_create_campaign_exports_table.php | 44 +++-------------- 4 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 app/Models/CampaignExport.php diff --git a/app/Jobs/Campaigns/Export.php b/app/Jobs/Campaigns/Export.php index 749335b6bc..801066f5c2 100644 --- a/app/Jobs/Campaigns/Export.php +++ b/app/Jobs/Campaigns/Export.php @@ -4,6 +4,7 @@ use App\Jobs\FileCleanup; use App\Models\Campaign; +use App\Models\CampaignExport; use App\Services\Campaign\ExportService; use App\User; use Exception; @@ -31,16 +32,19 @@ class Export implements ShouldQueue protected int $userId; + protected int $campaignExportId; + protected bool $assets; /** * CampaignExport constructor. */ - public function __construct(Campaign $campaign, User $user, bool $assets = false) + public function __construct(Campaign $campaign, User $user, CampaignExport $campaignExport, bool $assets = false) { $this->campaignId = $campaign->id; $this->userId = $user->id; $this->assets = $assets; + $this->campaignExportId = $campaignExport->id; } /** @@ -49,6 +53,12 @@ public function __construct(Campaign $campaign, User $user, bool $assets = false */ public function handle() { + $campaignExport = CampaignExport::find($this->campaignExportId); + if (!$campaignExport) { + return 0; + } + $campaignExport->update(['status' => CampaignExport::STATUS_RUNNING]); + /** @var Campaign|null $campaign */ $campaign = Campaign::find($this->campaignId); if (!$campaign) { @@ -63,7 +73,7 @@ public function handle() /** @var ExportService $service */ $service = app()->make(ExportService::class); - $service + $filesize = $service ->user($user) ->campaign($campaign) ->assets($this->assets) @@ -74,7 +84,7 @@ public function handle() if ($queue !== 'sync') { FileCleanup::dispatch($service->exportPath())->delay(now()->addMinutes(60)); } - + $campaignExport->update(['status' => CampaignExport::STATUS_FINISHED, 'size' => $filesize]); return 1; } @@ -83,6 +93,12 @@ public function handle() */ public function failed(Exception $exception) { + $campaignExport = CampaignExport::find($this->campaignExportId); + if (!$campaignExport) { + return; + } + $campaignExport->update(['status' => CampaignExport::STATUS_FAILED]); + // Set the campaign export date to null so that the user can try again. // If it failed once, trying again won't help, but this might motivate // them to report the error. diff --git a/app/Models/CampaignExport.php b/app/Models/CampaignExport.php new file mode 100644 index 0000000000..a6d670b728 --- /dev/null +++ b/app/Models/CampaignExport.php @@ -0,0 +1,48 @@ +subDays(90)); + } + +} diff --git a/app/Services/Campaign/ExportService.php b/app/Services/Campaign/ExportService.php index d6f304c3c7..715253d6b3 100644 --- a/app/Services/Campaign/ExportService.php +++ b/app/Services/Campaign/ExportService.php @@ -5,6 +5,7 @@ use App\Facades\CampaignCache; use App\Jobs\Campaigns\Export; use App\Models\Image; +use App\Models\CampaignExport; use App\Notifications\Header; use App\Traits\CampaignAware; use App\Traits\UserAware; @@ -29,6 +30,7 @@ class ExportService protected bool $assets = false; protected int $files = 0; + protected int $filesize = 0; public function exportPath(): string { @@ -46,13 +48,28 @@ public function queue(): self $this->campaign->export_date = date('Y-m-d'); $this->campaign->saveQuietly(); - Export::dispatch($this->campaign, $this->user, false); - Export::dispatch($this->campaign, $this->user, true); + $entitiesExport = CampaignExport::create([ + 'campaign_id' => $this->campaign->id, + 'created_by' => $this->user->id, + 'type' => CampaignExport::TYPE_ENTITIES, + 'status' => CampaignExport::STATUS_SCHEDULED, + ]); + + Export::dispatch($this->campaign, $this->user, $entitiesExport, false); + + $assetExport = CampaignExport::create([ + 'campaign_id' => $this->campaign->id, + 'created_by' => $this->user->id, + 'type' => CampaignExport::TYPE_ASSETS, + 'status' => CampaignExport::STATUS_SCHEDULED, + ]); + + Export::dispatch($this->campaign, $this->user, $assetExport, true); return $this; } - public function export(): self + public function export(): int { $this ->prepare() @@ -63,7 +80,7 @@ public function export(): self ->notify() ; - return $this; + return $this->filesize; } protected function prepare(): self @@ -212,6 +229,8 @@ protected function finish(): self $saveFolder = storage_path() . '/exports/campaigns/'; $this->archive->saveTo($saveFolder); + $this->filesize = (int) floor(filesize($this->path)/ pow(1024, 2)); + } catch (Exception $e) { // The export might fail if the zip is too big. $this->files = 0; diff --git a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php index 0b08aefce9..c8bfc6f09e 100644 --- a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php +++ b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php @@ -11,49 +11,18 @@ */ public function up(): void { - - - $test = 'Create a new table `campaign_exports` that has an id, - campaign_id, - created_by, - size, - type tinyint, - timestamps, tinyint status. - Campaign/user foreign key. - Index on status. - User id needs to be nullable, for when we start the process in the console. - Instead of cascadeOnDelete, use nullOnDelete instead - (so someone who deletes their data, we still have a trace of the export)'; - Schema::create('campaign_exports', function (Blueprint $table) { - $table->id(); + $table->increments('id'); $table->unsignedInteger('campaign_id'); - - - $table->timestamps(); $table->unsignedInteger('created_by')->nullable(); - $table->unsignedInteger('updated_by')->nullable(); - - $table->foreign('created_by')->references('id')->on('users')->onDelete('set null'); - $table->foreign('updated_by')->references('id')->on('users')->onDelete('set null'); - - $table->mediumText('config')->nullable(); - - $table->unsignedInteger('family_id'); - $table->foreign('family_id')->references('id')->on('families')->cascadeOnDelete(); - - - - $table->bigIncrements('id'); - $table->unsignedInteger('campaign_id'); - $table->unsignedInteger('created_by')->nullable(); - - $table->string('name', 100); - - $table->index('campaign_id'); + $table->integer('size'); + $table->tinyInteger('type'); + $table->tinyInteger('status'); $table->timestamps(); + $table->index(['status']); + $table->foreign('campaign_id')->references('id')->on('campaigns')->onDelete('cascade'); $table->foreign('created_by')->references('id')->on('users')->onDelete('set null'); @@ -66,6 +35,5 @@ public function up(): void public function down(): void { Schema::dropIfExists('campaign_exports'); - } }; From d983a95368ca3d43abd40eca2373d97bd067b43d Mon Sep 17 00:00:00 2001 From: Spitfire Date: Fri, 27 Oct 2023 11:59:27 -0600 Subject: [PATCH 3/5] Fixed Typo --- app/Models/CampaignExport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Models/CampaignExport.php b/app/Models/CampaignExport.php index a6d670b728..c1f44d7fc8 100644 --- a/app/Models/CampaignExport.php +++ b/app/Models/CampaignExport.php @@ -27,7 +27,7 @@ class CampaignExport extends Model public const STATUS_SCHEDULED = 1; public const STATUS_RUNNING = 2; public const STATUS_FINISHED = 3; - public const STATUS_FAILED = 3; + public const STATUS_FAILED = 4; public $fillable = [ 'size', From cfaa0c06ef65c18e9783c0be5101150a885b4b5c Mon Sep 17 00:00:00 2001 From: spitfire305 Date: Fri, 27 Oct 2023 18:25:21 +0000 Subject: [PATCH 4/5] Fix styling --- app/Models/CampaignExport.php | 3 +-- app/Services/Campaign/ExportService.php | 4 ++-- .../2023_10_25_173253_create_campaign_exports_table.php | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/Models/CampaignExport.php b/app/Models/CampaignExport.php index c1f44d7fc8..f806a3f205 100644 --- a/app/Models/CampaignExport.php +++ b/app/Models/CampaignExport.php @@ -3,7 +3,6 @@ namespace App\Models; use Illuminate\Database\Eloquent\Builder; -use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\MassPrunable; use Illuminate\Database\Eloquent\Model; @@ -15,7 +14,7 @@ * @property int $campaign_id * @property int $created_by * @property int $status - * + * */ class CampaignExport extends Model { diff --git a/app/Services/Campaign/ExportService.php b/app/Services/Campaign/ExportService.php index 715253d6b3..f9e5127aff 100644 --- a/app/Services/Campaign/ExportService.php +++ b/app/Services/Campaign/ExportService.php @@ -63,7 +63,7 @@ public function queue(): self 'type' => CampaignExport::TYPE_ASSETS, 'status' => CampaignExport::STATUS_SCHEDULED, ]); - + Export::dispatch($this->campaign, $this->user, $assetExport, true); return $this; @@ -229,7 +229,7 @@ protected function finish(): self $saveFolder = storage_path() . '/exports/campaigns/'; $this->archive->saveTo($saveFolder); - $this->filesize = (int) floor(filesize($this->path)/ pow(1024, 2)); + $this->filesize = (int) floor(filesize($this->path) / pow(1024, 2)); } catch (Exception $e) { // The export might fail if the zip is too big. diff --git a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php index c8bfc6f09e..de2d4912fd 100644 --- a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php +++ b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php @@ -4,8 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -return new class extends Migration -{ +return new class () extends Migration { /** * Run the migrations. */ From 2d88e6ab4e0c7a976110868d650f854e04ebaf6f Mon Sep 17 00:00:00 2001 From: Spitfire Date: Mon, 30 Oct 2023 15:16:00 -0600 Subject: [PATCH 5/5] Fixed code --- app/Jobs/Campaigns/Export.php | 5 +++-- app/Models/CampaignExport.php | 4 +++- app/Services/Campaign/ExportService.php | 7 ++++++- .../2023_10_25_173253_create_campaign_exports_table.php | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/Jobs/Campaigns/Export.php b/app/Jobs/Campaigns/Export.php index 801066f5c2..e70f1bb256 100644 --- a/app/Jobs/Campaigns/Export.php +++ b/app/Jobs/Campaigns/Export.php @@ -73,18 +73,19 @@ public function handle() /** @var ExportService $service */ $service = app()->make(ExportService::class); - $filesize = $service + $service ->user($user) ->campaign($campaign) ->assets($this->assets) ->export(); + $campaignExport->update(['status' => CampaignExport::STATUS_FINISHED, 'size' => $service->filesize(), 'path' => $service->exportPath()]); + // Don't delete in "sync" mode as there is no delay. $queue = config('queue.default'); if ($queue !== 'sync') { FileCleanup::dispatch($service->exportPath())->delay(now()->addMinutes(60)); } - $campaignExport->update(['status' => CampaignExport::STATUS_FINISHED, 'size' => $filesize]); return 1; } diff --git a/app/Models/CampaignExport.php b/app/Models/CampaignExport.php index f806a3f205..4928517a8d 100644 --- a/app/Models/CampaignExport.php +++ b/app/Models/CampaignExport.php @@ -14,6 +14,7 @@ * @property int $campaign_id * @property int $created_by * @property int $status + * @property string $path * */ class CampaignExport extends Model @@ -33,7 +34,8 @@ class CampaignExport extends Model 'type', 'status', 'campaign_id', - 'created_by' + 'created_by', + 'path', ]; /** diff --git a/app/Services/Campaign/ExportService.php b/app/Services/Campaign/ExportService.php index f9e5127aff..cb6a1d2bce 100644 --- a/app/Services/Campaign/ExportService.php +++ b/app/Services/Campaign/ExportService.php @@ -69,7 +69,7 @@ public function queue(): self return $this; } - public function export(): int + public function export(): self { $this ->prepare() @@ -80,6 +80,11 @@ public function export(): int ->notify() ; + return $this; + } + + public function filesize(): int + { return $this->filesize; } diff --git a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php index de2d4912fd..ec839b30a1 100644 --- a/database/migrations/2023_10_25_173253_create_campaign_exports_table.php +++ b/database/migrations/2023_10_25_173253_create_campaign_exports_table.php @@ -18,6 +18,7 @@ public function up(): void $table->integer('size'); $table->tinyInteger('type'); $table->tinyInteger('status'); + $table->string('path')->nullable(); $table->timestamps(); $table->index(['status']);