Better upload system using parallelisme

photologue/models.py

@@ -521,7 +521,7 @@ class ImageModel(models.Model):
 
 
 class Photo(ImageModel):
-    title = models.CharField(_("title"), max_length=250, unique=True)
+    title = models.CharField(_("title"), max_length=250)
     slug = models.SlugField(
         _("slug"),
         unique=True,

photologue/static/upload.js

@@ -5,27 +5,26 @@
 // When user drags files, register them in the file field
 const dropZone = document.getElementById('drop_zone');
 const uploadInput = document.getElementById('id_file_field');
-const form = document.getElementById('upload_form')
+const form = document.getElementById('upload_form');
 
 dropZone.ondrop = function (e) {
     e.preventDefault();
     this.className = 'upload-drop-zone';
-    console.log(e.dataTransfer.files)
     uploadInput.files = e.dataTransfer.files;
-}
+};
 
 dropZone.ondragover = function () {
     this.className = 'upload-drop-zone drop';
     return false;
-}
+};
 
 dropZone.ondragleave = function () {
     this.className = 'upload-drop-zone';
     return false;
-}
+};
 
 // When user selects an existing gallery, disable new gallery fields
-const gallerySelect = document.getElementById('id_gallery')
+const gallerySelect = document.getElementById('id_gallery');
 gallerySelectUpdate = () => {
     const useGallery = (gallerySelect.value !== "");
     document.getElementById('id_new_gallery_title').disabled = useGallery;
@@ -33,149 +32,199 @@ gallerySelectUpdate = () => {
     document.getElementById('id_new_gallery_date_end').disabled = useGallery;
     document.getElementById('id_new_gallery_tags').disabled = useGallery;
     document.getElementById('id_new_gallery_description').disabled = useGallery;
-}
+};
 gallerySelect.addEventListener('change', gallerySelectUpdate);
 gallerySelectUpdate();
 
-// On submit, show a message to make user wait
+// Hide basic submit button
-document.getElementById('upload_form').addEventListener('submit', (e) => {
+const submitbtn = document.getElementById('submit-id-submit');
-    document.getElementById('submit-id-submit').disabled = true;
+submitbtn.style.display = 'none';
-    document.getElementById('submit-id-submit').value = "Please be patient";
-});
 
-submitbtn = document.getElementById('submit-id-submit');
+// Create upload + pause buttons in a flex row
-//submitbtn.type = "button";
+const btnWrap = document.createElement('div');
+btnWrap.className = 'd-flex align-items-center gap-2 mt-2';
 
-//submitbtn.addEventListener("click", uplaodfnc);
+const ctnbtn = document.createElement('button');
+ctnbtn.className = submitbtn.className.replace(/\bmt-\d\b/g, '').trim();
+ctnbtn.type = 'button';
+ctnbtn.textContent = gettext('Upload');
 
-var ctnbtn = document.createElement("input");
+const pausebtn = document.createElement('button');
-ctnbtn.classList = submitbtn.classList;
+pausebtn.className = 'btn btn-warning';
-ctnbtn.value=gettext("Continious Upload");
+pausebtn.type = 'button';
-ctnbtn.type = "button";
+pausebtn.textContent = gettext('Pause');
+pausebtn.style.display = 'none';
 
-var pause = false ;
+btnWrap.appendChild(ctnbtn);
+btnWrap.appendChild(pausebtn);
+form.appendChild(btnWrap);
+
+// Create progress bar
+const progressWrap = document.createElement('div');
+progressWrap.className = 'progress mt-3';
+progressWrap.style.display = 'none';
+const progressBar = document.createElement('div');
+progressBar.className = 'progress-bar progress-bar-striped progress-bar-animated';
+progressBar.setAttribute('role', 'progressbar');
+progressBar.style.width = '0%';
+progressWrap.appendChild(progressBar);
+
+// Concurrency label
+const concurrencyLabel = document.createElement('small');
+concurrencyLabel.className = 'text-muted mt-1';
+concurrencyLabel.style.display = 'none';
+
+form.appendChild(progressWrap);
+form.appendChild(concurrencyLabel);
+
+let pause = false;
+let uploading = false;
+
+function beforeUnloadHandler(e) {
+    e.preventDefault();
+}
+window.addEventListener('beforeunload', (e) => { if (uploading) beforeUnloadHandler(e); });
 
 function sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+pausebtn.addEventListener('click', () => {
+    pause = !pause;
+    pausebtn.textContent = pause ? gettext('Resume') : gettext('Pause');
+    pausebtn.className = pause ? 'btn btn-success' : 'btn btn-warning';
+});
 
-async function pausing() {
+async function uploadFile(file, galleryID, csrfvalue) {
-    pause = ! pause ;
+    const sendform = new FormData();
-    if (pause) {
+    sendform.append('csrfmiddlewaretoken', csrfvalue);
-        ctnbtn.value = gettext("Pausing upload");
+    sendform.append('file_field', file);
-    } else {
+    sendform.append('gallery', galleryID);
-        ctnbtn.value = gettext("Resume");
+    const start = performance.now();
+    const res = await fetch('/upload/', {
+        method: 'POST',
+        body: sendform,
+        headers: { 'Accept': 'application/json' }
+    });
+    if (!res.ok) {
+        let detail = '';
+        try { const j = await res.json(); detail = j.error || ''; } catch (_) {}
+        console.error('Error with ' + file.name + ' (HTTP ' + res.status + ')' + (detail ? ': ' + JSON.stringify(detail) : ''));
     }
-    
+    return performance.now() - start;
-    
 }
 
-async function uplaodfnc() {
+ctnbtn.addEventListener('click', async () => {
-
+    const files = Array.from(uploadInput.files);
-    ctnbtn.removeEventListener("click",uplaodfnc)
+    if (!files.length) {
-    ctnbtn.addEventListener("click",pausing)
+        window.alert(gettext('Please select files first.'));
-
+        return;
-    console.log(uploadInput.files);
-    files = uploadInput.files;
-    submitbtn.disabled = true;
-    //ctnbtn.disabled = true;
-    actual = 0;
-    ctnbtn.value = gettext("Pause the upload, be patient")+" 0% (" + actual + "/" + files.length + ")";
-
-    csrfvalue = document.getElementsByName("csrfmiddlewaretoken")[0].value;
-
-    gallery_ID = gallerySelect.value;
-
-    if (gallery_ID == "") {
-        // Create gallery
     }
 
-    total = files.length;
+    uploading = true;
+    ctnbtn.disabled = true;
 
-    fdata = new FormData(uploadInput.form);
+    const csrfvalue = document.getElementsByName('csrfmiddlewaretoken')[0].value;
-    fdata.delete("file_field");
+    const total = files.length;
-    //fdata.append("reptype", "json");
+    let completed = 0;
 
-
+    // Create gallery / validate form
-    response = await fetch("/upload/", {
+    const fdata = new FormData(uploadInput.form);
-        method: "POST",
+    fdata.delete('file_field');
+    const response = await fetch('/upload/', {
+        method: 'POST',
         body: fdata,
-        headers: {
+        headers: { 'Accept': 'application/json' }
-            'Accept': 'application/json'
-        }
     });
-    returned = await response.json();
+    const returned = await response.json();
 
     if (returned.code != 200) {
-        window.alert("There is an error in the form" + returned.error);
+        const msgs = Object.entries(returned.error).map(([f, e]) => f + ': ' + e.join(', ')).join('\n');
+        window.alert(msgs);
+        ctnbtn.disabled = false;
+        return;
     }
 
-    // Upload files
+    const galleryID = returned.galleryID;
-    for (let file of files) {
-        while (pause){
-            await sleep(100);
-            ctnbtn.value = gettext("Upload Paused") + " " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")";
-        }
-        actual++;
-        sendform = new FormData();
-        sendform.append("csrfmiddlewaretoken", csrfvalue);
-        sendform.append("file_field", file);
-        //sendform.append("reptype", "json");
-        sendform.append("gallery", returned.galleryID);
 
-        try {
+    uploadInput.disabled = true;
-            const response = await fetch("/upload/", {
+    gallerySelect.disabled = true;
-                method: "POST",
+    dropZone.style.pointerEvents = 'none';
-                body: sendform,
+    pausebtn.style.display = 'inline-block';
-                headers: {
+    progressWrap.style.display = 'flex';
-            'Accept': 'application/json'
+    concurrencyLabel.style.display = 'block';
-        }
-            });
-            okpass = await response.ok;
-            if (!okpass) {
-                window.alert("Error with " + file.name + "code" + await response.code);
-            }
-            if (!pause) {
-                ctnbtn.value = gettext("Pause the upload, be patient")+" " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")";
-            }
-            
 
-        } catch (e) {
+    // Auto-tuning state
-            console.error(e);
+    const MIN_CONCURRENCY = 1;
-        }
+    const MAX_CONCURRENCY = 6;
+    let concurrency = 1;
+    let lastRate = null;
+    let lastMeasureTime = performance.now();
+    let lastCompleted = 0;
 
+    function updateProgress() {
+        const pct = Math.round(100 * completed / total);
+        progressBar.style.width = pct + '%';
+        progressBar.textContent = pct + '% (' + completed + '/' + total + ')';
+        concurrencyLabel.textContent = gettext('Parallel uploads:') + ' ' + concurrency;
     }
 
-    fdata = new FormData(uploadInput.form);
+    function tuneConcurrency() {
-    fdata.delete("file_field");
+        const now = performance.now();
-    fdata.delete("new_gallery_title");
+        const elapsed = now - lastMeasureTime;
-    fdata.delete("new_galleru_date_start");
+        if (elapsed < 1000) return;
-    fdata.delete("new_galleru_date_end");
+        const rate = (completed - lastCompleted) / elapsed;
-    //fdata.append("reptype", "json");
+        lastMeasureTime = now;
-    fdata.append("gallery", returned.galleryID);
+        lastCompleted = completed;
-    fdata.append("end", "end")
+        if (lastRate === null || rate > lastRate * 1.05)
+            concurrency = Math.min(concurrency + 1, MAX_CONCURRENCY);
+        else if (rate < lastRate * 0.90)
+            concurrency = Math.max(concurrency - 1, MIN_CONCURRENCY);
+        lastRate = rate;
+    }
 
-    response = await fetch("/upload/", {
+    // Parallel worker pool
-        method: "POST",
+    const queue = [...files];
-        body: fdata,
+    let active = 0;
-        headers: {
+
-            'Accept': 'application/json'
+    await new Promise((resolve) => {
+        function next() {
+            if (completed === total) { resolve(); return; }
+
+            while (!pause && active < concurrency && queue.length > 0) {
+                const file = queue.shift();
+                active++;
+                uploadFile(file, galleryID, csrfvalue)
+                    .catch(e => console.error(e))
+                    .finally(() => { completed++; tuneConcurrency(); active--; updateProgress(); next(); });
+            }
+
+            if (pause && active === 0 && queue.length > 0)
+                setTimeout(next, 200);
         }
+        next();
     });
-    returned = await response.json();
-
-
-    ctnbtn.disabled = true;
-    ctnbtn.value = gettext("Upload Complete Please reload the page");
-}
-
-
-ctnbtn.addEventListener("click", uplaodfnc);
-
-form.appendChild(ctnbtn);
-
-
-
 
+    // Finalize
+    const fend = new FormData(uploadInput.form);
+    fend.delete('file_field');
+    fend.delete('new_gallery_title');
+    fend.delete('new_gallery_date_start');
+    fend.delete('new_gallery_date_end');
+    fend.append('gallery', galleryID);
+    fend.append('end', 'end');
+    await fetch('/upload/', {
+        method: 'POST',
+        body: fend,
+        headers: { 'Accept': 'application/json' }
+    });
 
+    uploading = false;
+    uploadInput.disabled = false;
+    gallerySelect.disabled = false;
+    dropZone.style.pointerEvents = '';
+    pausebtn.style.display = 'none';
+    progressBar.classList.remove('progress-bar-animated');
+    progressBar.textContent = gettext('Upload complete — please reload the page');
+    concurrencyLabel.textContent = gettext('Best concurrency found:') + ' ' + concurrency;
+});

photologue/views.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
 from django.core.mail import mail_admins
-from django.db import IntegrityError
+from django.db import transaction
 from django.http import HttpResponse
 from django.http import JsonResponse
 from django.shortcuts import get_object_or_404, redirect
@@ -275,6 +275,13 @@ class GalleryUpload(PermissionRequiredMixin, FormView):
     success_url = reverse_lazy("photologue:pl-gallery-upload")
     permission_required = "photologue.add_gallery"
 
+    def form_invalid(self, form):
+        if not self.request.accepts("text/html") and self.request.accepts("application/json"):
+            errors = {field: list(errs) for field, errs in form.errors.items()}
+            return JsonResponse({"code": 400, "error": errors}, status=400)
+        return super().form_invalid(form)
+
+
     def form_valid(self, form):
         # Get or create gallery
 
@@ -312,22 +319,42 @@ class GalleryUpload(PermissionRequiredMixin, FormView):
                 jsondata["error"] = f"{photo_file.name} was not recognized as an image"
                 continue
 
-            title = f"{gallery.title} - {photo_file.name}"
+            title = Path(photo_file.name).stem
-            try:
+
-                photo = Photo(
+            already_exist = Photo.objects.filter(
-                    title=title,
+                title=title,
-                    slug=slugify(title),
+                owner=self.request.user,
-                    owner=self.request.user,
+                galleries=gallery
-                )
+            ).exists()
-                photo_name = str(gallery_dir / photo_file.name)
+
+            if already_exist:
+                already_exists += 1
+                continue
+
+            # Find a uniq slug
+            base_slug = slugify(title)
+            slug = base_slug
+            counter = 2
+            while Photo.objects.filter(slug=slug).exists():
+                slug = f"{base_slug}-{counter}"
+                counter += 1
+            photo = Photo(
+                title=title,
+                slug=slug,
+                owner=self.request.user,
+            )
+            photo_name = str(gallery_dir / photo_file.name)
+
+            # Save photo and associate it with the gallery in a single database operation
+            # Defer image file saving until the database commit succeeds
+            with transaction.atomic():
                 photo.save()
                 photo.galleries.set([gallery])
 
-                # Save to disk after successful database edit
+                def save_file():
-                photo.image.save(photo_name, photo_file)
+                    photo.image.save(photo_name, photo_file)
-            except IntegrityError:
+
-                already_exists += 1
+                transaction.on_commit(save_file)
-                continue
 
             uploaded_photo_name.append(photo_file.name)