diff --git a/photologue/models.py b/photologue/models.py index 99a6449..5f97cf7 100644 --- a/photologue/models.py +++ b/photologue/models.py @@ -521,7 +521,7 @@ class ImageModel(models.Model): class Photo(ImageModel): - title = models.CharField(_("title"), max_length=250, unique=True) + title = models.CharField(_("title"), max_length=250) slug = models.SlugField( _("slug"), unique=True, diff --git a/photologue/static/upload.js b/photologue/static/upload.js index 74bcd2e..c879213 100644 --- a/photologue/static/upload.js +++ b/photologue/static/upload.js @@ -5,27 +5,26 @@ // When user drags files, register them in the file field const dropZone = document.getElementById('drop_zone'); const uploadInput = document.getElementById('id_file_field'); -const form = document.getElementById('upload_form') +const form = document.getElementById('upload_form'); dropZone.ondrop = function (e) { e.preventDefault(); this.className = 'upload-drop-zone'; - console.log(e.dataTransfer.files) uploadInput.files = e.dataTransfer.files; -} +}; dropZone.ondragover = function () { this.className = 'upload-drop-zone drop'; return false; -} +}; dropZone.ondragleave = function () { this.className = 'upload-drop-zone'; return false; -} +}; // When user selects an existing gallery, disable new gallery fields -const gallerySelect = document.getElementById('id_gallery') +const gallerySelect = document.getElementById('id_gallery'); gallerySelectUpdate = () => { const useGallery = (gallerySelect.value !== ""); document.getElementById('id_new_gallery_title').disabled = useGallery; @@ -33,149 +32,199 @@ gallerySelectUpdate = () => { document.getElementById('id_new_gallery_date_end').disabled = useGallery; document.getElementById('id_new_gallery_tags').disabled = useGallery; document.getElementById('id_new_gallery_description').disabled = useGallery; -} +}; gallerySelect.addEventListener('change', gallerySelectUpdate); gallerySelectUpdate(); -// On submit, show a message to make user wait -document.getElementById('upload_form').addEventListener('submit', (e) => { - document.getElementById('submit-id-submit').disabled = true; - document.getElementById('submit-id-submit').value = "Please be patient"; -}); +// Hide basic submit button +const submitbtn = document.getElementById('submit-id-submit'); +submitbtn.style.display = 'none'; -submitbtn = document.getElementById('submit-id-submit'); -//submitbtn.type = "button"; +// Create upload + pause buttons in a flex row +const btnWrap = document.createElement('div'); +btnWrap.className = 'd-flex align-items-center gap-2 mt-2'; -//submitbtn.addEventListener("click", uplaodfnc); +const ctnbtn = document.createElement('button'); +ctnbtn.className = submitbtn.className.replace(/\bmt-\d\b/g, '').trim(); +ctnbtn.type = 'button'; +ctnbtn.textContent = gettext('Upload'); -var ctnbtn = document.createElement("input"); -ctnbtn.classList = submitbtn.classList; -ctnbtn.value=gettext("Continious Upload"); -ctnbtn.type = "button"; +const pausebtn = document.createElement('button'); +pausebtn.className = 'btn btn-warning'; +pausebtn.type = 'button'; +pausebtn.textContent = gettext('Pause'); +pausebtn.style.display = 'none'; -var pause = false ; +btnWrap.appendChild(ctnbtn); +btnWrap.appendChild(pausebtn); +form.appendChild(btnWrap); + +// Create progress bar +const progressWrap = document.createElement('div'); +progressWrap.className = 'progress mt-3'; +progressWrap.style.display = 'none'; +const progressBar = document.createElement('div'); +progressBar.className = 'progress-bar progress-bar-striped progress-bar-animated'; +progressBar.setAttribute('role', 'progressbar'); +progressBar.style.width = '0%'; +progressWrap.appendChild(progressBar); + +// Concurrency label +const concurrencyLabel = document.createElement('small'); +concurrencyLabel.className = 'text-muted mt-1'; +concurrencyLabel.style.display = 'none'; + +form.appendChild(progressWrap); +form.appendChild(concurrencyLabel); + +let pause = false; +let uploading = false; + +function beforeUnloadHandler(e) { + e.preventDefault(); +} +window.addEventListener('beforeunload', (e) => { if (uploading) beforeUnloadHandler(e); }); function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } +pausebtn.addEventListener('click', () => { + pause = !pause; + pausebtn.textContent = pause ? gettext('Resume') : gettext('Pause'); + pausebtn.className = pause ? 'btn btn-success' : 'btn btn-warning'; +}); -async function pausing() { - pause = ! pause ; - if (pause) { - ctnbtn.value = gettext("Pausing upload"); - } else { - ctnbtn.value = gettext("Resume"); +async function uploadFile(file, galleryID, csrfvalue) { + const sendform = new FormData(); + sendform.append('csrfmiddlewaretoken', csrfvalue); + sendform.append('file_field', file); + sendform.append('gallery', galleryID); + const start = performance.now(); + const res = await fetch('/upload/', { + method: 'POST', + body: sendform, + headers: { 'Accept': 'application/json' } + }); + if (!res.ok) { + let detail = ''; + try { const j = await res.json(); detail = j.error || ''; } catch (_) {} + console.error('Error with ' + file.name + ' (HTTP ' + res.status + ')' + (detail ? ': ' + JSON.stringify(detail) : '')); } - - + return performance.now() - start; } -async function uplaodfnc() { - - ctnbtn.removeEventListener("click",uplaodfnc) - ctnbtn.addEventListener("click",pausing) - - console.log(uploadInput.files); - files = uploadInput.files; - submitbtn.disabled = true; - //ctnbtn.disabled = true; - actual = 0; - ctnbtn.value = gettext("Pause the upload, be patient")+" 0% (" + actual + "/" + files.length + ")"; - - csrfvalue = document.getElementsByName("csrfmiddlewaretoken")[0].value; - - gallery_ID = gallerySelect.value; - - if (gallery_ID == "") { - // Create gallery +ctnbtn.addEventListener('click', async () => { + const files = Array.from(uploadInput.files); + if (!files.length) { + window.alert(gettext('Please select files first.')); + return; } - total = files.length; + uploading = true; + ctnbtn.disabled = true; - fdata = new FormData(uploadInput.form); - fdata.delete("file_field"); - //fdata.append("reptype", "json"); + const csrfvalue = document.getElementsByName('csrfmiddlewaretoken')[0].value; + const total = files.length; + let completed = 0; - - response = await fetch("/upload/", { - method: "POST", + // Create gallery / validate form + const fdata = new FormData(uploadInput.form); + fdata.delete('file_field'); + const response = await fetch('/upload/', { + method: 'POST', body: fdata, - headers: { - 'Accept': 'application/json' - } + headers: { 'Accept': 'application/json' } }); - returned = await response.json(); + const returned = await response.json(); if (returned.code != 200) { - window.alert("There is an error in the form" + returned.error); + const msgs = Object.entries(returned.error).map(([f, e]) => f + ': ' + e.join(', ')).join('\n'); + window.alert(msgs); + ctnbtn.disabled = false; + return; } - // Upload files - for (let file of files) { - while (pause){ - await sleep(100); - ctnbtn.value = gettext("Upload Paused") + " " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")"; - } - actual++; - sendform = new FormData(); - sendform.append("csrfmiddlewaretoken", csrfvalue); - sendform.append("file_field", file); - //sendform.append("reptype", "json"); - sendform.append("gallery", returned.galleryID); + const galleryID = returned.galleryID; - try { - const response = await fetch("/upload/", { - method: "POST", - body: sendform, - headers: { - 'Accept': 'application/json' - } - }); - okpass = await response.ok; - if (!okpass) { - window.alert("Error with " + file.name + "code" + await response.code); - } - if (!pause) { - ctnbtn.value = gettext("Pause the upload, be patient")+" " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")"; - } - + uploadInput.disabled = true; + gallerySelect.disabled = true; + dropZone.style.pointerEvents = 'none'; + pausebtn.style.display = 'inline-block'; + progressWrap.style.display = 'flex'; + concurrencyLabel.style.display = 'block'; - } catch (e) { - console.error(e); - } + // Auto-tuning state + const MIN_CONCURRENCY = 1; + const MAX_CONCURRENCY = 6; + let concurrency = 1; + let lastRate = null; + let lastMeasureTime = performance.now(); + let lastCompleted = 0; + function updateProgress() { + const pct = Math.round(100 * completed / total); + progressBar.style.width = pct + '%'; + progressBar.textContent = pct + '% (' + completed + '/' + total + ')'; + concurrencyLabel.textContent = gettext('Parallel uploads:') + ' ' + concurrency; } - fdata = new FormData(uploadInput.form); - fdata.delete("file_field"); - fdata.delete("new_gallery_title"); - fdata.delete("new_galleru_date_start"); - fdata.delete("new_galleru_date_end"); - //fdata.append("reptype", "json"); - fdata.append("gallery", returned.galleryID); - fdata.append("end", "end") + function tuneConcurrency() { + const now = performance.now(); + const elapsed = now - lastMeasureTime; + if (elapsed < 1000) return; + const rate = (completed - lastCompleted) / elapsed; + lastMeasureTime = now; + lastCompleted = completed; + if (lastRate === null || rate > lastRate * 1.05) + concurrency = Math.min(concurrency + 1, MAX_CONCURRENCY); + else if (rate < lastRate * 0.90) + concurrency = Math.max(concurrency - 1, MIN_CONCURRENCY); + lastRate = rate; + } - response = await fetch("/upload/", { - method: "POST", - body: fdata, - headers: { - 'Accept': 'application/json' + // Parallel worker pool + const queue = [...files]; + let active = 0; + + await new Promise((resolve) => { + function next() { + if (completed === total) { resolve(); return; } + + while (!pause && active < concurrency && queue.length > 0) { + const file = queue.shift(); + active++; + uploadFile(file, galleryID, csrfvalue) + .catch(e => console.error(e)) + .finally(() => { completed++; tuneConcurrency(); active--; updateProgress(); next(); }); + } + + if (pause && active === 0 && queue.length > 0) + setTimeout(next, 200); } + next(); }); - returned = await response.json(); - - - ctnbtn.disabled = true; - ctnbtn.value = gettext("Upload Complete Please reload the page"); -} - - -ctnbtn.addEventListener("click", uplaodfnc); - -form.appendChild(ctnbtn); - - - + // Finalize + const fend = new FormData(uploadInput.form); + fend.delete('file_field'); + fend.delete('new_gallery_title'); + fend.delete('new_gallery_date_start'); + fend.delete('new_gallery_date_end'); + fend.append('gallery', galleryID); + fend.append('end', 'end'); + await fetch('/upload/', { + method: 'POST', + body: fend, + headers: { 'Accept': 'application/json' } + }); + uploading = false; + uploadInput.disabled = false; + gallerySelect.disabled = false; + dropZone.style.pointerEvents = ''; + pausebtn.style.display = 'none'; + progressBar.classList.remove('progress-bar-animated'); + progressBar.textContent = gettext('Upload complete — please reload the page'); + concurrencyLabel.textContent = gettext('Best concurrency found:') + ' ' + concurrency; +}); diff --git a/photologue/views.py b/photologue/views.py index bfd7641..c44b1f8 100644 --- a/photologue/views.py +++ b/photologue/views.py @@ -12,7 +12,7 @@ from pathlib import Path from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from django.core.mail import mail_admins -from django.db import IntegrityError +from django.db import transaction from django.http import HttpResponse from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect @@ -275,6 +275,13 @@ class GalleryUpload(PermissionRequiredMixin, FormView): success_url = reverse_lazy("photologue:pl-gallery-upload") permission_required = "photologue.add_gallery" + def form_invalid(self, form): + if not self.request.accepts("text/html") and self.request.accepts("application/json"): + errors = {field: list(errs) for field, errs in form.errors.items()} + return JsonResponse({"code": 400, "error": errors}, status=400) + return super().form_invalid(form) + + def form_valid(self, form): # Get or create gallery @@ -312,22 +319,42 @@ class GalleryUpload(PermissionRequiredMixin, FormView): jsondata["error"] = f"{photo_file.name} was not recognized as an image" continue - title = f"{gallery.title} - {photo_file.name}" - try: - photo = Photo( - title=title, - slug=slugify(title), - owner=self.request.user, - ) - photo_name = str(gallery_dir / photo_file.name) + title = Path(photo_file.name).stem + + already_exist = Photo.objects.filter( + title=title, + owner=self.request.user, + galleries=gallery + ).exists() + + if already_exist: + already_exists += 1 + continue + + # Find a uniq slug + base_slug = slugify(title) + slug = base_slug + counter = 2 + while Photo.objects.filter(slug=slug).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + photo = Photo( + title=title, + slug=slug, + owner=self.request.user, + ) + photo_name = str(gallery_dir / photo_file.name) + + # Save photo and associate it with the gallery in a single database operation + # Defer image file saving until the database commit succeeds + with transaction.atomic(): photo.save() photo.galleries.set([gallery]) - # Save to disk after successful database edit - photo.image.save(photo_name, photo_file) - except IntegrityError: - already_exists += 1 - continue + def save_file(): + photo.image.save(photo_name, photo_file) + + transaction.on_commit(save_file) uploaded_photo_name.append(photo_file.name)