Django S3 Pre Signed URLs

How much memory does my server use uploading files?

Say you are creating a voice recording app and a user records 30 minutes audio clip and sends it to your django server. How much system memory do you eat up?

The browsers media capture will record in its favored compressed audio format and send about a 30MB compressed audio blob:
– Safari: Usually produces MP4 with AAC codec
– Chrome/Edge: Usually produces WebM with Opus codec
– Firefox: Usually produces OGG with Opus codec

Django request.FILES contains this file and will be in memory as its accessed and uploaded to S3.
– 1 user uploading 30MB = 30MB
– 10 user uploading 30MB = 300MB
– 100 user uploading 30MB = 3GB
🚨 Do you feel your Heroku hobby dyno straining?

Allow the client to upload to S3 directly

Q: How could that be safe?
Note: The client should never hold credentials for your S3.
However, your server can request a special expiring url (pre-signed url) to pass to the user, allowing the user to upload files directly to your bucket.

Code

Bash
# this includes boto3
pip install django-storages[s3]
JavaScript
async function uploadRecording(audioBlob) {
    // Step 1: Get presigned URL from your server
    const response = await fetch('/generate-presigned-url/', {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
            'X-CSRFToken': csrfToken
        },
        body: JSON.stringify({
            mime_type: audioBlob.type  // e.g., "audio/webm;codecs=opus"
        })
    });

    const { upload_url, upload_fields, s3_key } = await response.json();

    // Step 2: Upload directly to S3
    const s3FormData = new FormData();
    Object.keys(upload_fields).forEach(key => {
        s3FormData.append(key, upload_fields[key]);
    });
    s3FormData.append('file', audioBlob);

    await fetch(upload_url, {
        method: 'POST',
        body: s3FormData
    });

    // Step 3: Confirm upload with your server
    await fetch('/confirm-upload/', {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
            'X-CSRFToken': csrfToken
        },
        body: JSON.stringify({ s3_key })
    });
}
Python
# s3_utils.py
import boto3
import uuid
from django.conf import settings
from django.utils import timezone
from botocore.exceptions import ClientError

def get_s3_client():
    """Get an S3 client configured with Django settings."""
    return boto3.client(
        's3',
        aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
        region_name=settings.AWS_S3_REGION_NAME
    )
    
def generate_presigned_post_for_recording(mime_type='audio/webm', file_extension='webm'):
    s3_client = get_s3_client()

    # Generate unique filename
    timestamp = timezone.now().strftime('%Y%m%d_%H%M%S')
    unique_id = uuid.uuid4().hex[:8]
    s3_key = f"media/audio_files/recording_{timestamp}_{unique_id}.{file_extension}"

    return s3_client.generate_presigned_post(
        Bucket=settings.AWS_STORAGE_BUCKET_NAME,
        Key=s3_key,
        ExpiresIn=3600,  # 1 hour expiration
        Conditions=[
            {'Content-Type': mime_type},
            ['content-length-range', 0, 100 * 1024 * 1024]  # Max 100MB
        ]
    )

# views.py
import json
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.utils import timezone

@login_required
def generate_presigned_url(request):
    data = json.loads(request.body)
    mime_type = data.get('mime_type', 'audio/webm')

    # Map MIME type to file extension
    mime_to_ext = {
        'audio/webm': 'webm',
        'audio/ogg': 'ogg',
        'audio/mp4': 'm4a',
    }
    file_extension = mime_to_ext.get(mime_type, 'webm')

    presigned_data = generate_presigned_post_for_recording(mime_type, file_extension)

    return JsonResponse({
        'upload_url': presigned_data['url'],
        'upload_fields': presigned_data['fields'],
        's3_key': presigned_data['s3_key']
    })

def verify_s3_upload(s3_key):
    """Verify that a file was successfully uploaded to S3."""
    s3_client = get_s3_client()  # You'll need this helper too
    try:
        s3_client.head_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key)
        return True
    except ClientError:
        return False

@login_required
def confirm_s3_upload(request):
    data = json.loads(request.body)
    s3_key = data.get('s3_key')

    # Verify file exists in S3
    if verify_s3_upload(s3_key):
        # Create Note record pointing to S3 file
        note = Note.objects.create(
            user=request.user,
            title=f"Recording {timezone.now().strftime('%Y-%m-%d')}",
        )
        # Set file path (Django storages handles S3 URLs automatically)
        note.audio_file.name = s3_key.replace('media/', '', 1)
        note.save()

        return JsonResponse({'id': note.id})

    return JsonResponse({'error': 'Upload failed'}, status=400)


# urls.py
path('generate-presigned-url/', views.generate_presigned_url, name='generate_presigned_url'),
path('confirm-upload/', views.confirm_s3_upload, name='confirm_s3_upload'),
JavaScript
  //  S3 CORS Configuration Required

[
    {
        "AllowedHeaders": ["*"],
        "AllowedMethods": ["PUT", "POST"],
        "AllowedOrigins": [
            "https://your-domain.com",
            "http://localhost:8000"
        ],
        "ExposeHeaders": ["ETag"],
        "MaxAgeSeconds": 3000
    }
]
Bash
## Required Environment Variables
DJANGO_AWS_ACCESS_KEY_ID=your_key
DJANGO_AWS_SECRET_ACCESS_KEY=your_secret
DJANGO_AWS_STORAGE_BUCKET_NAME=your-bucket
DJANGO_AWS_S3_REGION_NAME=us-east-1

Results

Before (Traditional Upload):
– Browser → Django → S3
– Memory: 30MB per upload held in Django RAM

After (Direct Upload):
– Browser → S3 (Django only handles metadata)
– Memory: ~0MB Django RAM usage for file transfer