diff --git a/core/__init__.py b/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core/admin.py b/core/admin.py deleted file mode 100644 index 64b4591..0000000 --- a/core/admin.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.contrib import admin - -from .models import FileObject, Post, S3Object - -admin.site.register(Post) -admin.site.register(S3Object) diff --git a/core/apps.py b/core/apps.py deleted file mode 100644 index 8115ae6..0000000 --- a/core/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class CoreConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'core' diff --git a/core/bucket.py b/core/bucket.py deleted file mode 100644 index 380424c..0000000 --- a/core/bucket.py +++ /dev/null @@ -1,88 +0,0 @@ -import logging - -import boto3 -from botocore.exceptions import BotoCoreError, ClientError -from django.conf import settings -from rest_framework import status -from rest_framework.response import Response - -from django_tus.response import Tus404, TusResponse - -logger = logging.getLogger(__name__) - - -class S3MultipartUploader: - def __init__(self, *args, **kwargs): - self.bucket_name = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME - self.access_key = settings.MINIO_STORAGE_ACCESS_KEY - self.secret_key = settings.MINIO_STORAGE_SECRET_KEY - self.endpoint_url = f'https://{settings.MINIO_STORAGE_ENDPOINT}' - self.region_name = '' - self.s3 = self._create_s3_client() - - def _create_s3_client(self): - return boto3.client( - 's3', - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - endpoint_url=self.endpoint_url, - ) - - def generate_multipart_upload(self, filename, content_type, metadata): - # Initialize a multipart upload - response = self.s3.create_multipart_upload( - Bucket=self.bucket_name, - Key=filename, - ACL="public-read-write", - ContentType=content_type, - CacheControl="max-age=1000", - Metadata=metadata, - ) - upload_id = response['UploadId'] - - return upload_id - - def parts_upload( - self, tmp_path: str, file_name: str, file_size: int, upload_id: str - ): - try: - # Determine part size and number of parts - part_size = 5 * 1024 * 1024 # 5MB - file_size = file_size - num_parts = (file_size + part_size - 1) // part_size - - # Upload parts - parts = [] - with open(tmp_path, 'r+b') as f: - for part_number in range(1, num_parts + 1): - data = f.read(part_size) - - response = self.s3.upload_part( - Bucket=self.bucket_name, - Key=file_name, - PartNumber=part_number, - UploadId=upload_id, - Body=data, - ) - parts.append({'PartNumber': part_number, 'ETag': response['ETag']}) - - return parts - except (BotoCoreError, ClientError) as e: - # Handle the exception here - print("Error uploading parts:", e) - return None - - def complete_upload(self, parts: list, upload_id: str, file_name: str): - try: - completeResult = self.s3.complete_multipart_upload( - Bucket=self.bucket_name, - Key=file_name, - UploadId=upload_id, - MultipartUpload={'Parts': parts}, - ) - - return completeResult - except (BotoCoreError, ClientError) as e: - # Handle the exception here - print("Error completing multipart upload:", e) - return None diff --git a/core/forms.py b/core/forms.py deleted file mode 100644 index c3f2578..0000000 --- a/core/forms.py +++ /dev/null @@ -1,11 +0,0 @@ -from django import forms - -from .models import Post - - -class PostForm(forms.ModelForm): - """Post form""" - - class Meta: - model = Post - fields = '__all__' diff --git a/core/metadata.py b/core/metadata.py deleted file mode 100644 index 66acf04..0000000 --- a/core/metadata.py +++ /dev/null @@ -1,18 +0,0 @@ -import base64 - -from rest_framework.metadata import BaseMetadata - - -class CustomMetadata(BaseMetadata): - def determine_metadata(self, request, view): - metadata = {} - - upload_metadata = request.META.get("HTTP_UPLOAD_METADATA") - - if upload_metadata: - for kv in upload_metadata.split(","): - key, value = kv.split(" ", 1) if " " in kv else (kv, "") - decoded_value = base64.b64decode(value).decode() - metadata[key] = decoded_value - - return metadata diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py deleted file mode 100644 index e54b878..0000000 --- a/core/migrations/0001_initial.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-10 01:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Post', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('title', models.CharField(blank=True, max_length=100)), - ('file', models.FileField(upload_to='')), - ], - ), - ] diff --git a/core/migrations/0002_fileobject.py b/core/migrations/0002_fileobject.py deleted file mode 100644 index df601d0..0000000 --- a/core/migrations/0002_fileobject.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-12 07:51 - -import core.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='FileObject', - fields=[ - ('id', models.CharField(db_index=True, default=core.models.generate_random_string, editable=False, max_length=16, primary_key=True, serialize=False, unique=True, verbose_name='File ID')), - ('filename', models.CharField(blank=True, max_length=255, verbose_name='File Name')), - ('length', models.BigIntegerField(default=-1, verbose_name='File Length')), - ('offset', models.BigIntegerField(default=0, verbose_name='Offset')), - ('metadata', models.JSONField(default=dict, verbose_name='Metadata')), - ('tmp_path', models.CharField(max_length=4096, null=True, verbose_name='Temporary Path')), - ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created At')), - ('expires_at', models.DateTimeField(blank=True, null=True, verbose_name='Expiration Date')), - ('file', models.FileField(upload_to='uploads/', verbose_name='File')), - ], - options={ - 'verbose_name': 'File', - 'verbose_name_plural': 'Files', - }, - ), - ] diff --git a/core/migrations/0003_fileobject_image_fileobject_video.py b/core/migrations/0003_fileobject_image_fileobject_video.py deleted file mode 100644 index fdf9d57..0000000 --- a/core/migrations/0003_fileobject_image_fileobject_video.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-21 11:10 - -import cloudinary_storage.storage -import cloudinary_storage.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0002_fileobject'), - ] - - operations = [ - migrations.AddField( - model_name='fileobject', - name='image', - field=models.ImageField(blank=True, upload_to='images/'), - ), - migrations.AddField( - model_name='fileobject', - name='video', - field=models.ImageField(blank=True, storage=cloudinary_storage.storage.VideoMediaCloudinaryStorage(), upload_to='videos/', validators=[cloudinary_storage.validators.validate_video]), - ), - ] diff --git a/core/migrations/0004_s3object.py b/core/migrations/0004_s3object.py deleted file mode 100644 index 7c022e8..0000000 --- a/core/migrations/0004_s3object.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-21 11:16 - -import cloudinary_storage.storage -import cloudinary_storage.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0003_fileobject_image_fileobject_video'), - ] - - operations = [ - migrations.CreateModel( - name='S3Object', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('video', models.ImageField(blank=True, storage=cloudinary_storage.storage.VideoMediaCloudinaryStorage(), upload_to='videos/', validators=[cloudinary_storage.validators.validate_video])), - ('image', models.ImageField(blank=True, upload_to='images/')), - ('raw_file', models.ImageField(blank=True, storage=cloudinary_storage.storage.RawMediaCloudinaryStorage(), upload_to='raw/')), - ], - ), - ] diff --git a/core/migrations/0005_s3object_file.py b/core/migrations/0005_s3object_file.py deleted file mode 100644 index 6bf3a7b..0000000 --- a/core/migrations/0005_s3object_file.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-21 11:28 - -import cloudinary.models -from django.db import migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0004_s3object'), - ] - - operations = [ - migrations.AddField( - model_name='s3object', - name='file', - field=cloudinary.models.CloudinaryField(default=django.utils.timezone.now, max_length=255), - preserve_default=False, - ), - ] diff --git a/core/migrations/0006_remove_s3object_image_remove_s3object_raw_file_and_more.py b/core/migrations/0006_remove_s3object_image_remove_s3object_raw_file_and_more.py deleted file mode 100644 index aec808f..0000000 --- a/core/migrations/0006_remove_s3object_image_remove_s3object_raw_file_and_more.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-21 13:21 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0005_s3object_file'), - ] - - operations = [ - migrations.RemoveField( - model_name='s3object', - name='image', - ), - migrations.RemoveField( - model_name='s3object', - name='raw_file', - ), - migrations.RemoveField( - model_name='s3object', - name='video', - ), - ] diff --git a/core/migrations/__init__.py b/core/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core/models.py b/core/models.py deleted file mode 100644 index 12c9274..0000000 --- a/core/models.py +++ /dev/null @@ -1,88 +0,0 @@ -import random - -from cloudinary.models import CloudinaryField -from cloudinary_storage.storage import ( - RawMediaCloudinaryStorage, - VideoMediaCloudinaryStorage, -) -from cloudinary_storage.validators import validate_video -from django.db import models -from django.utils import timezone - - -def generate_random_string(length=16): - """Generate a random string of given length.""" - characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - return ''.join(random.choice(characters) for _ in range(length)) - - -class Post(models.Model): - title = models.CharField(max_length=100, blank=True) - file = models.FileField() - - def __str__(self): - return self.title - - -class FileObject(models.Model): - """Default model for a tus file upload.""" - - id = models.CharField( - max_length=16, - default=generate_random_string, - unique=True, - primary_key=True, - db_index=True, - verbose_name="File ID", - editable=False, - ) - video = models.ImageField( - upload_to='videos/', - blank=True, - storage=VideoMediaCloudinaryStorage(), - validators=[validate_video], - ) - image = models.ImageField( - upload_to='images/', blank=True - ) # no need to set storage, field will use the default one - filename = models.CharField(max_length=255, blank=True, verbose_name="File Name") - length = models.BigIntegerField(default=-1, verbose_name="File Length") - offset = models.BigIntegerField(default=0, verbose_name="Offset") - metadata = models.JSONField(default=dict, verbose_name="Metadata") - tmp_path = models.CharField( - max_length=4096, null=True, verbose_name="Temporary Path" - ) - created_at = models.DateTimeField(auto_now_add=True, verbose_name="Created At") - expires_at = models.DateTimeField( - null=True, blank=True, verbose_name="Expiration Date" - ) - file = models.FileField(upload_to='uploads/', verbose_name="File") - - def __str__(self): - return self.filename or self.id - - class Meta: - verbose_name = "File" - verbose_name_plural = "Files" - - def is_expired(self): - """Check if the file has expired.""" - if self.expires_at: - return timezone.now() > self.expires_at - return False - - is_expired.boolean = True - is_expired.short_description = "Expired" - - @property - def url(self): - return self.file.url - - @property - def size_in_kb(self): - """Return the size of the file in kilobytes.""" - return round(self.length / 1024, 2) - - -class S3Object(models.Model): - file = CloudinaryField(resource_type='auto') diff --git a/core/process.py b/core/process.py deleted file mode 100644 index 44f2c96..0000000 --- a/core/process.py +++ /dev/null @@ -1,121 +0,0 @@ -import io -import logging -import os -import uuid - -from django.conf import settings -from django.core.cache import cache -from django.core.files.storage import default_storage -from rest_framework import status - -from .response import Tus404, TusResponse - -logger = logging.getLogger(__name__) - - -class FileResource: - def __init__(self, resource_id): - self.resource_id = resource_id - self.client = default_storage.client - self.bucket = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME - self._load_file_info() - - def _load_file_info(self): - self.filename = cache.get(f"uploads/{self.resource_id}/filename") - self.file_size = int(cache.get(f"uploads/{self.resource_id}/file_size")) - self.metadata = cache.get(f"uploads/{self.resource_id}/metadata") - self.offset = cache.get(f"uploads/{self.resource_id}/offset") - - @staticmethod - def get_file_or_404(resource_id): - print('resource get or 404: ', resource_id) - - if FileResource.resource_exists(str(resource_id)): - return FileResource(resource_id) - else: - raise Tus404() - - @staticmethod - def resource_exists(resource_id: str): - print('Resource Exist: ', resource_id) - return cache.get(f"uploads/{resource_id}/filename", None) is not None - - @staticmethod - def create_initial_file(metadata, file_size: int): - resource_id = str(uuid.uuid4()) - - filename_key = f"uploads/{resource_id}/filename" - cache.add(filename_key, f"{metadata.get('filename')}") - - file_size_key = f"uploads/{resource_id}/file_size" - cache.add(file_size_key, file_size) - - offset_key = f"uploads/{resource_id}/offset" - cache.add(offset_key, 0) - - metadata_key = f"uploads/{resource_id}/metadata" - cache.add(metadata_key, metadata) - - file = FileResource(resource_id) - file.write_init_file() - - return file - - def get_path(self): - return os.path.join(settings.TUS_UPLOAD_DIR, self.resource_id) - - def write_init_file(self): - try: - object_name = f"uploads/{self.resource_id}/{self.filename}" - - self.client.put_object( - self.bucket, - object_name, - io.BytesIO(b'\0'), - 1, - ) - except IOError as e: - error_message = f"Unable to create file: {e}" - logger.error(error_message, exc_info=True) - return TusResponse( - {'error': 'Internal server error'}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - - def write_chunk(self, chunk): - try: - object_name = f"uploads/{self.resource_id}/{self.filename}" - offset_key = f"uploads/{self.resource_id}/offset" - chunk_offset = cache.get(offset_key, default=0) - - # Upload the chunk to MinIO - self.client.put_object( - self.bucket, - object_name, - io.BytesIO(chunk.content), - chunk.chunk_size, - part_number=chunk.chunk_number, - ) - - # Update the offset in cache - cache.set(offset_key, chunk_offset + chunk.chunk_size) - - except IOError as e: - logger.error( - "write_chunk", - extra={ - 'request': chunk.META, - 'tus': { - "resource_id": self.resource_id, - "filename": self.filename, - "file_size": self.file_size, - "metadata": self.metadata, - "offset": self.offset, - "upload_file_path": self.get_path(), - }, - }, - ) - return TusResponse( - {'error': 'Unable to write chunk'}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) diff --git a/core/response.py b/core/response.py deleted file mode 100644 index c8d555e..0000000 --- a/core/response.py +++ /dev/null @@ -1,43 +0,0 @@ -import re - -from rest_framework.exceptions import NotFound -from rest_framework.response import Response - -tus_api_version = '1.0.0' -tus_api_version_supported = ['1.0.0'] -tus_api_extensions = ['creation', 'termination', 'file-check'] - - -class TusResponse(Response): - def __init__(self, headers=None, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.base_tus_headers = { - 'Tus-Resumable': tus_api_version, - 'Tus-Version': ",".join(tus_api_version_supported), - 'Tus-Extension': ",".join(tus_api_extensions), - # 'Tus-Max-Size': settings.TUS_MAX_FILE_SIZE, TODO: Need to be checked the file size - 'Access-Control-Allow-Origin': "*", - # 'Access-Control-Allow-Methods': "PATCH,HEAD,GET,POST,OPTIONS", - 'Access-Control-Expose-Headers': "Tus-Resumable,upload-length,upload-metadata,Location,Upload-Offset", - 'Access-Control-Allow-Headers': "Tus-Resumable,upload-length,upload-metadata,Location,Upload-Offset,content-type", - 'Cache-Control': 'no-store', - } - - self.sanitize_headers(self.base_tus_headers) - if headers: - self.sanitize_headers(headers) - - def sanitize_headers(self, headers: dict): - for key, value in headers.items(): - if not re.match(r'^[a-zA-Z0-9_-]+$', key): - raise ValueError(f'Invalid header name: {key}') - - if not value: - raise ValueError(f'Invalid header value: {value}') - - self.__setitem__(key, value) - - -class Tus404(TusResponse, NotFound): - pass diff --git a/core/serializers.py b/core/serializers.py deleted file mode 100644 index cd4fc74..0000000 --- a/core/serializers.py +++ /dev/null @@ -1,11 +0,0 @@ -from rest_framework import serializers - -from .models import FileObject - - -class FileObjectSerializer(serializers.ModelSerializer): - """File Serializer""" - - class Meta: - model = FileObject - fields = '__all__' diff --git a/core/urls.py b/core/urls.py deleted file mode 100644 index c96cc4c..0000000 --- a/core/urls.py +++ /dev/null @@ -1,12 +0,0 @@ -from re import I - -from django.urls import path -from django.views.generic import TemplateView - -from .views import FileUploaderApi, PostView - -urlpatterns = [ - path('', PostView.as_view(), name='post_view'), - path('files/', FileUploaderApi.as_view(), name='file_upload_api'), - path('uppy/', TemplateView.as_view(template_name='uppy.html'), name="uppy"), -] diff --git a/core/views.py b/core/views.py deleted file mode 100644 index c31c2be..0000000 --- a/core/views.py +++ /dev/null @@ -1,85 +0,0 @@ -from django.shortcuts import redirect, render -from django.views.generic import View -from rest_framework import permissions, status -from rest_framework.views import APIView - -from django_tus.models import TusFileModel - -from .forms import PostForm -from .metadata import CustomMetadata -from .models import Post -from .process import FileResource -from .response import TusResponse - - -class PostView(View): - template_name = 'tus.html' - form_class = PostForm - - def get(self, request): - post_query = TusFileModel.objects.all() - form = self.form_class() - - context = {'posts': post_query, 'form': form} - return render(request, self.template_name, context) - - def post(self, request, *args, **kwargs): - form = self.form_class(request.POST) - if form.is_valid(): - # - form.save() - - return redirect('/') - - return render(request, self.template_name, {"form": form}) - - -class FileUploaderApi(APIView): - """Create file API View""" - - permission_classes = [permissions.AllowAny] - metadata_class = CustomMetadata - - def get_metadata(self, request): - meta = self.metadata_class() - return meta.determine_metadata(request, self) - - def options(self, request, *args, **kwargs): - return TusResponse(status=status.HTTP_204_NO_CONTENT) - - def post(self, request, *args, **kwargs): - metadata = self.get_metadata(request) - - content_type = metadata.get('filetype', None) - file_size = int(request.META.get("HTTP_UPLOAD_LENGTH", "0")) - - file = FileResource.create_initial_file(metadata, file_size) - - return TusResponse( - status=status.HTTP_201_CREATED, - # headers={'Location': '{}'.format(object_url)}, - ) - - def head(self, request, resource_id): - file = FileResource.get_file_or_404(str(resource_id)) - print('Head: ', file) - return TusResponse( - status=status.HTTP_200_OK, - headers={ - 'Upload-Offset': file.offset, - 'Upload-Length': file.file_size, - }, - ) - - def patch(self, request, resource_id, *args, **kwargs): - file = FileResource.get_file_or_404(str(resource_id)) - - return TusResponse(status=status.HTTP_204_NO_CONTENT) - - -class Chunk: - def __init__(self, request): - self.META = request.META - self.chunk_number = int(request.META.get('HTTP_UPLOAD_OFFSET', 0)) - self.chunk_size = int(request.META.get("CONTENT_LENGTH", 102400)) - self.content = request.body diff --git a/django_tus/tasks.py b/django_tus/tasks.py deleted file mode 100644 index 8cfc6eb..0000000 --- a/django_tus/tasks.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -from pathlib import Path - -from celery import Task, shared_task -from django.conf import settings -from django.core.files.storage import default_storage -from django.shortcuts import get_object_or_404 - -# from django_tus.bucket import S3MultipartUploader -from django_tus.models import TusFileModel - - -class BaseTaskWithRetry(Task): - autoretry_for = (Exception, KeyError) - retry_kwargs = {'max_retries': 3} - retry_backoff = True - - -def extract_uuid(input_string): - parts = input_string.split('/') - if len(parts) == 2: - return parts[1] - else: - return None - - -def final_path(filename): - return os.path.join(settings.TUS_DESTINATION_DIR, filename) - - -@shared_task(base=BaseTaskWithRetry) -def file_load_to_bucket(temp_path, filename, file_size, upload_id, resource_id): - # s3 = S3MultipartUploader() - - # parts = s3.parts_upload(temp_path, filename, file_size, upload_id) - - # complete = s3.complete_upload(parts, upload_id, filename) - # location = complete.get('Location', '') - - # os.remove(temp_path) - - file_path = final_path(filename) - - store_file = get_object_or_404(TusFileModel, guid=resource_id) - # store_file.uploaded_file = location - # Save the uploaded file to the path specified in the FileField's upload_to argument - print('uploading...') - print(file_path) - default_storage.save(store_file.uploaded_file, file_path) - # store_file.save() - print('Uploading Done!') diff --git a/server/settings.py b/server/settings.py index 9bfb7e5..87e4efc 100644 --- a/server/settings.py +++ b/server/settings.py @@ -36,14 +36,11 @@ 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', - 'cloudinary_storage', 'django.contrib.staticfiles', - 'cloudinary', 'corsheaders', 'rest_framework', 'minio_storage', 'django_tus', - 'core', ] MIDDLEWARE = [ @@ -145,8 +142,6 @@ # global config DEFAULT_FILE_STORAGE = "minio_storage.storage.MinioMediaStorage" -# DEFAULT_FILE_STORAGE = 'cloudinary_storage.storage.MediaCloudinaryStorage' -# STATICFILES_STORAGE = 'cloudinary_storage.storage.StaticHashedCloudinaryStorage' # STATICFILES_STORAGE = "minio_storage.storage.MinioStaticStorage" MINIO_STORAGE_ENDPOINT = 'bucket.cb.media:9000' @@ -191,13 +186,6 @@ } -CLOUDINARY_STORAGE = { - 'CLOUD_NAME': 'drrmwbyx7', - 'API_KEY': '183796315324614', - 'API_SECRET': 'Kimk5VV6XVfUdvdmuWpyolgux1k', -} - - # Celery settings CELERY_BROKER_URL = "redis://localhost:6379" CELERY_RESULT_BACKEND = "redis://localhost:6379" diff --git a/server/urls.py b/server/urls.py index a62b2b6..23fe48c 100644 --- a/server/urls.py +++ b/server/urls.py @@ -21,7 +21,6 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('', include('core.urls')), path('', include('django_tus.urls')), ]