Merge pull request #219 from lzaoral/fix-large-uploads

Fix large file uploads

kobo/client/__init__.py

@@ -470,6 +470,10 @@ def upload_file(self, file_name, target_dir):
         checksum = sum.hexdigest().lower()
 
         fsize = os.path.getsize(file_name)
+        # use str only for large uploads to not break compatibility with older hubs
+        if fsize > xmlrpclib.MAXINT:
+            fsize = str(fsize)
+
         upload_id, upload_key = self.upload.register_upload(os.path.basename(file_name), checksum, fsize, target_dir)
 
         secure = (scheme == "https")

kobo/django/upload/migrations/0002_alter_fileupload_size.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.20 on 2023-08-22 09:45
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('upload', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='fileupload',
+            name='size',
+            field=models.BigIntegerField(validators=[django.core.validators.MinValueValidator(0)]),
+        ),
+    ]

kobo/django/upload/models.py

@@ -4,6 +4,7 @@
 import os
 
 from django.conf import settings
+from django.core.validators import MinValueValidator
 from django.db import models
 from django.contrib.auth.models import User
 
@@ -25,7 +26,8 @@ class FileUpload(models.Model):
     owner       = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
     name        = models.CharField(max_length=255)
     checksum    = models.CharField(max_length=255)
-    size        = models.PositiveIntegerField()
+    # models.PositiveBigIntegerField would be even better but it was introduced only in Django 3.1
+    size        = models.BigIntegerField(validators=[MinValueValidator(0)])
     target_dir = models.CharField(max_length=255)
     upload_key  = models.CharField(max_length=255)
     state       = models.PositiveIntegerField(default=0, choices=UPLOAD_STATES.get_mapping())
@@ -66,6 +68,9 @@ def save(self, *args, **kwargs):
                 self.state = UPLOAD_STATES['FAILED']
                 if "update_fields" in kwargs:
                     kwargs["update_fields"] = {"state"}.union(kwargs["update_fields"])
+
+        # execute validators
+        self.full_clean()
         super(FileUpload, self).save(*args, **kwargs)
 
     def delete(self):

kobo/django/upload/xmlrpc.py

@@ -27,7 +27,8 @@ def register_upload(request, name, checksum, size, target_dir):
     upload.owner = request.user
     upload.name = name
     upload.checksum = checksum.lower()
-    upload.size = size
+    # size may be sent as a string to workaround the xmlrpc.client.MAXINT limit
+    upload.size = int(size)
     upload.target_dir = target_dir
     upload.save()
     return (upload.id, upload.upload_key)