Better upload system using parallelisme
This commit is contained in:
krek0
parent
debf87421f
commit
f221740228
3 changed files with 207 additions and 131 deletions
|
|
@ -521,7 +521,7 @@ class ImageModel(models.Model):
|
|||
|
||||
|
||||
class Photo(ImageModel):
|
||||
title = models.CharField(_("title"), max_length=250, unique=True)
|
||||
title = models.CharField(_("title"), max_length=250)
|
||||
slug = models.SlugField(
|
||||
_("slug"),
|
||||
unique=True,
|
||||
|
|
|
|||
|
|
@ -5,27 +5,26 @@
|
|||
// When user drags files, register them in the file field
|
||||
const dropZone = document.getElementById('drop_zone');
|
||||
const uploadInput = document.getElementById('id_file_field');
|
||||
const form = document.getElementById('upload_form')
|
||||
const form = document.getElementById('upload_form');
|
||||
|
||||
dropZone.ondrop = function (e) {
|
||||
e.preventDefault();
|
||||
this.className = 'upload-drop-zone';
|
||||
console.log(e.dataTransfer.files)
|
||||
uploadInput.files = e.dataTransfer.files;
|
||||
}
|
||||
};
|
||||
|
||||
dropZone.ondragover = function () {
|
||||
this.className = 'upload-drop-zone drop';
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
dropZone.ondragleave = function () {
|
||||
this.className = 'upload-drop-zone';
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// When user selects an existing gallery, disable new gallery fields
|
||||
const gallerySelect = document.getElementById('id_gallery')
|
||||
const gallerySelect = document.getElementById('id_gallery');
|
||||
gallerySelectUpdate = () => {
|
||||
const useGallery = (gallerySelect.value !== "");
|
||||
document.getElementById('id_new_gallery_title').disabled = useGallery;
|
||||
|
|
@ -33,149 +32,199 @@ gallerySelectUpdate = () => {
|
|||
document.getElementById('id_new_gallery_date_end').disabled = useGallery;
|
||||
document.getElementById('id_new_gallery_tags').disabled = useGallery;
|
||||
document.getElementById('id_new_gallery_description').disabled = useGallery;
|
||||
}
|
||||
};
|
||||
gallerySelect.addEventListener('change', gallerySelectUpdate);
|
||||
gallerySelectUpdate();
|
||||
|
||||
// On submit, show a message to make user wait
|
||||
document.getElementById('upload_form').addEventListener('submit', (e) => {
|
||||
document.getElementById('submit-id-submit').disabled = true;
|
||||
document.getElementById('submit-id-submit').value = "Please be patient";
|
||||
});
|
||||
// Hide basic submit button
|
||||
const submitbtn = document.getElementById('submit-id-submit');
|
||||
submitbtn.style.display = 'none';
|
||||
|
||||
submitbtn = document.getElementById('submit-id-submit');
|
||||
//submitbtn.type = "button";
|
||||
// Create upload + pause buttons in a flex row
|
||||
const btnWrap = document.createElement('div');
|
||||
btnWrap.className = 'd-flex align-items-center gap-2 mt-2';
|
||||
|
||||
//submitbtn.addEventListener("click", uplaodfnc);
|
||||
const ctnbtn = document.createElement('button');
|
||||
ctnbtn.className = submitbtn.className.replace(/\bmt-\d\b/g, '').trim();
|
||||
ctnbtn.type = 'button';
|
||||
ctnbtn.textContent = gettext('Upload');
|
||||
|
||||
var ctnbtn = document.createElement("input");
|
||||
ctnbtn.classList = submitbtn.classList;
|
||||
ctnbtn.value=gettext("Continious Upload");
|
||||
ctnbtn.type = "button";
|
||||
const pausebtn = document.createElement('button');
|
||||
pausebtn.className = 'btn btn-warning';
|
||||
pausebtn.type = 'button';
|
||||
pausebtn.textContent = gettext('Pause');
|
||||
pausebtn.style.display = 'none';
|
||||
|
||||
var pause = false ;
|
||||
btnWrap.appendChild(ctnbtn);
|
||||
btnWrap.appendChild(pausebtn);
|
||||
form.appendChild(btnWrap);
|
||||
|
||||
// Create progress bar
|
||||
const progressWrap = document.createElement('div');
|
||||
progressWrap.className = 'progress mt-3';
|
||||
progressWrap.style.display = 'none';
|
||||
const progressBar = document.createElement('div');
|
||||
progressBar.className = 'progress-bar progress-bar-striped progress-bar-animated';
|
||||
progressBar.setAttribute('role', 'progressbar');
|
||||
progressBar.style.width = '0%';
|
||||
progressWrap.appendChild(progressBar);
|
||||
|
||||
// Concurrency label
|
||||
const concurrencyLabel = document.createElement('small');
|
||||
concurrencyLabel.className = 'text-muted mt-1';
|
||||
concurrencyLabel.style.display = 'none';
|
||||
|
||||
form.appendChild(progressWrap);
|
||||
form.appendChild(concurrencyLabel);
|
||||
|
||||
let pause = false;
|
||||
let uploading = false;
|
||||
|
||||
function beforeUnloadHandler(e) {
|
||||
e.preventDefault();
|
||||
}
|
||||
window.addEventListener('beforeunload', (e) => { if (uploading) beforeUnloadHandler(e); });
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
pausebtn.addEventListener('click', () => {
|
||||
pause = !pause;
|
||||
pausebtn.textContent = pause ? gettext('Resume') : gettext('Pause');
|
||||
pausebtn.className = pause ? 'btn btn-success' : 'btn btn-warning';
|
||||
});
|
||||
|
||||
async function pausing() {
|
||||
pause = ! pause ;
|
||||
if (pause) {
|
||||
ctnbtn.value = gettext("Pausing upload");
|
||||
} else {
|
||||
ctnbtn.value = gettext("Resume");
|
||||
async function uploadFile(file, galleryID, csrfvalue) {
|
||||
const sendform = new FormData();
|
||||
sendform.append('csrfmiddlewaretoken', csrfvalue);
|
||||
sendform.append('file_field', file);
|
||||
sendform.append('gallery', galleryID);
|
||||
const start = performance.now();
|
||||
const res = await fetch('/upload/', {
|
||||
method: 'POST',
|
||||
body: sendform,
|
||||
headers: { 'Accept': 'application/json' }
|
||||
});
|
||||
if (!res.ok) {
|
||||
let detail = '';
|
||||
try { const j = await res.json(); detail = j.error || ''; } catch (_) {}
|
||||
console.error('Error with ' + file.name + ' (HTTP ' + res.status + ')' + (detail ? ': ' + JSON.stringify(detail) : ''));
|
||||
}
|
||||
|
||||
|
||||
return performance.now() - start;
|
||||
}
|
||||
|
||||
async function uplaodfnc() {
|
||||
|
||||
ctnbtn.removeEventListener("click",uplaodfnc)
|
||||
ctnbtn.addEventListener("click",pausing)
|
||||
|
||||
console.log(uploadInput.files);
|
||||
files = uploadInput.files;
|
||||
submitbtn.disabled = true;
|
||||
//ctnbtn.disabled = true;
|
||||
actual = 0;
|
||||
ctnbtn.value = gettext("Pause the upload, be patient")+" 0% (" + actual + "/" + files.length + ")";
|
||||
|
||||
csrfvalue = document.getElementsByName("csrfmiddlewaretoken")[0].value;
|
||||
|
||||
gallery_ID = gallerySelect.value;
|
||||
|
||||
if (gallery_ID == "") {
|
||||
// Create gallery
|
||||
ctnbtn.addEventListener('click', async () => {
|
||||
const files = Array.from(uploadInput.files);
|
||||
if (!files.length) {
|
||||
window.alert(gettext('Please select files first.'));
|
||||
return;
|
||||
}
|
||||
|
||||
total = files.length;
|
||||
uploading = true;
|
||||
ctnbtn.disabled = true;
|
||||
|
||||
fdata = new FormData(uploadInput.form);
|
||||
fdata.delete("file_field");
|
||||
//fdata.append("reptype", "json");
|
||||
const csrfvalue = document.getElementsByName('csrfmiddlewaretoken')[0].value;
|
||||
const total = files.length;
|
||||
let completed = 0;
|
||||
|
||||
|
||||
response = await fetch("/upload/", {
|
||||
method: "POST",
|
||||
// Create gallery / validate form
|
||||
const fdata = new FormData(uploadInput.form);
|
||||
fdata.delete('file_field');
|
||||
const response = await fetch('/upload/', {
|
||||
method: 'POST',
|
||||
body: fdata,
|
||||
headers: {
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
headers: { 'Accept': 'application/json' }
|
||||
});
|
||||
returned = await response.json();
|
||||
const returned = await response.json();
|
||||
|
||||
if (returned.code != 200) {
|
||||
window.alert("There is an error in the form" + returned.error);
|
||||
const msgs = Object.entries(returned.error).map(([f, e]) => f + ': ' + e.join(', ')).join('\n');
|
||||
window.alert(msgs);
|
||||
ctnbtn.disabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Upload files
|
||||
for (let file of files) {
|
||||
while (pause){
|
||||
await sleep(100);
|
||||
ctnbtn.value = gettext("Upload Paused") + " " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")";
|
||||
}
|
||||
actual++;
|
||||
sendform = new FormData();
|
||||
sendform.append("csrfmiddlewaretoken", csrfvalue);
|
||||
sendform.append("file_field", file);
|
||||
//sendform.append("reptype", "json");
|
||||
sendform.append("gallery", returned.galleryID);
|
||||
const galleryID = returned.galleryID;
|
||||
|
||||
try {
|
||||
const response = await fetch("/upload/", {
|
||||
method: "POST",
|
||||
body: sendform,
|
||||
headers: {
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
});
|
||||
okpass = await response.ok;
|
||||
if (!okpass) {
|
||||
window.alert("Error with " + file.name + "code" + await response.code);
|
||||
}
|
||||
if (!pause) {
|
||||
ctnbtn.value = gettext("Pause the upload, be patient")+" " + Math.round(100 * actual / total) + "% (" + actual + "/" + files.length + ")";
|
||||
}
|
||||
|
||||
uploadInput.disabled = true;
|
||||
gallerySelect.disabled = true;
|
||||
dropZone.style.pointerEvents = 'none';
|
||||
pausebtn.style.display = 'inline-block';
|
||||
progressWrap.style.display = 'flex';
|
||||
concurrencyLabel.style.display = 'block';
|
||||
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
// Auto-tuning state
|
||||
const MIN_CONCURRENCY = 1;
|
||||
const MAX_CONCURRENCY = 6;
|
||||
let concurrency = 1;
|
||||
let lastRate = null;
|
||||
let lastMeasureTime = performance.now();
|
||||
let lastCompleted = 0;
|
||||
|
||||
function updateProgress() {
|
||||
const pct = Math.round(100 * completed / total);
|
||||
progressBar.style.width = pct + '%';
|
||||
progressBar.textContent = pct + '% (' + completed + '/' + total + ')';
|
||||
concurrencyLabel.textContent = gettext('Parallel uploads:') + ' ' + concurrency;
|
||||
}
|
||||
|
||||
fdata = new FormData(uploadInput.form);
|
||||
fdata.delete("file_field");
|
||||
fdata.delete("new_gallery_title");
|
||||
fdata.delete("new_galleru_date_start");
|
||||
fdata.delete("new_galleru_date_end");
|
||||
//fdata.append("reptype", "json");
|
||||
fdata.append("gallery", returned.galleryID);
|
||||
fdata.append("end", "end")
|
||||
function tuneConcurrency() {
|
||||
const now = performance.now();
|
||||
const elapsed = now - lastMeasureTime;
|
||||
if (elapsed < 1000) return;
|
||||
const rate = (completed - lastCompleted) / elapsed;
|
||||
lastMeasureTime = now;
|
||||
lastCompleted = completed;
|
||||
if (lastRate === null || rate > lastRate * 1.05)
|
||||
concurrency = Math.min(concurrency + 1, MAX_CONCURRENCY);
|
||||
else if (rate < lastRate * 0.90)
|
||||
concurrency = Math.max(concurrency - 1, MIN_CONCURRENCY);
|
||||
lastRate = rate;
|
||||
}
|
||||
|
||||
response = await fetch("/upload/", {
|
||||
method: "POST",
|
||||
body: fdata,
|
||||
headers: {
|
||||
'Accept': 'application/json'
|
||||
// Parallel worker pool
|
||||
const queue = [...files];
|
||||
let active = 0;
|
||||
|
||||
await new Promise((resolve) => {
|
||||
function next() {
|
||||
if (completed === total) { resolve(); return; }
|
||||
|
||||
while (!pause && active < concurrency && queue.length > 0) {
|
||||
const file = queue.shift();
|
||||
active++;
|
||||
uploadFile(file, galleryID, csrfvalue)
|
||||
.catch(e => console.error(e))
|
||||
.finally(() => { completed++; tuneConcurrency(); active--; updateProgress(); next(); });
|
||||
}
|
||||
|
||||
if (pause && active === 0 && queue.length > 0)
|
||||
setTimeout(next, 200);
|
||||
}
|
||||
next();
|
||||
});
|
||||
returned = await response.json();
|
||||
|
||||
|
||||
ctnbtn.disabled = true;
|
||||
ctnbtn.value = gettext("Upload Complete Please reload the page");
|
||||
}
|
||||
|
||||
|
||||
ctnbtn.addEventListener("click", uplaodfnc);
|
||||
|
||||
form.appendChild(ctnbtn);
|
||||
|
||||
|
||||
|
||||
|
||||
// Finalize
|
||||
const fend = new FormData(uploadInput.form);
|
||||
fend.delete('file_field');
|
||||
fend.delete('new_gallery_title');
|
||||
fend.delete('new_gallery_date_start');
|
||||
fend.delete('new_gallery_date_end');
|
||||
fend.append('gallery', galleryID);
|
||||
fend.append('end', 'end');
|
||||
await fetch('/upload/', {
|
||||
method: 'POST',
|
||||
body: fend,
|
||||
headers: { 'Accept': 'application/json' }
|
||||
});
|
||||
|
||||
uploading = false;
|
||||
uploadInput.disabled = false;
|
||||
gallerySelect.disabled = false;
|
||||
dropZone.style.pointerEvents = '';
|
||||
pausebtn.style.display = 'none';
|
||||
progressBar.classList.remove('progress-bar-animated');
|
||||
progressBar.textContent = gettext('Upload complete — please reload the page');
|
||||
concurrencyLabel.textContent = gettext('Best concurrency found:') + ' ' + concurrency;
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from pathlib import Path
|
|||
from django.contrib import messages
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
|
||||
from django.core.mail import mail_admins
|
||||
from django.db import IntegrityError
|
||||
from django.db import transaction
|
||||
from django.http import HttpResponse
|
||||
from django.http import JsonResponse
|
||||
from django.shortcuts import get_object_or_404, redirect
|
||||
|
|
@ -275,6 +275,13 @@ class GalleryUpload(PermissionRequiredMixin, FormView):
|
|||
success_url = reverse_lazy("photologue:pl-gallery-upload")
|
||||
permission_required = "photologue.add_gallery"
|
||||
|
||||
def form_invalid(self, form):
|
||||
if not self.request.accepts("text/html") and self.request.accepts("application/json"):
|
||||
errors = {field: list(errs) for field, errs in form.errors.items()}
|
||||
return JsonResponse({"code": 400, "error": errors}, status=400)
|
||||
return super().form_invalid(form)
|
||||
|
||||
|
||||
def form_valid(self, form):
|
||||
# Get or create gallery
|
||||
|
||||
|
|
@ -312,22 +319,42 @@ class GalleryUpload(PermissionRequiredMixin, FormView):
|
|||
jsondata["error"] = f"{photo_file.name} was not recognized as an image"
|
||||
continue
|
||||
|
||||
title = f"{gallery.title} - {photo_file.name}"
|
||||
try:
|
||||
photo = Photo(
|
||||
title=title,
|
||||
slug=slugify(title),
|
||||
owner=self.request.user,
|
||||
)
|
||||
photo_name = str(gallery_dir / photo_file.name)
|
||||
title = Path(photo_file.name).stem
|
||||
|
||||
already_exist = Photo.objects.filter(
|
||||
title=title,
|
||||
owner=self.request.user,
|
||||
galleries=gallery
|
||||
).exists()
|
||||
|
||||
if already_exist:
|
||||
already_exists += 1
|
||||
continue
|
||||
|
||||
# Find a uniq slug
|
||||
base_slug = slugify(title)
|
||||
slug = base_slug
|
||||
counter = 2
|
||||
while Photo.objects.filter(slug=slug).exists():
|
||||
slug = f"{base_slug}-{counter}"
|
||||
counter += 1
|
||||
photo = Photo(
|
||||
title=title,
|
||||
slug=slug,
|
||||
owner=self.request.user,
|
||||
)
|
||||
photo_name = str(gallery_dir / photo_file.name)
|
||||
|
||||
# Save photo and associate it with the gallery in a single database operation
|
||||
# Defer image file saving until the database commit succeeds
|
||||
with transaction.atomic():
|
||||
photo.save()
|
||||
photo.galleries.set([gallery])
|
||||
|
||||
# Save to disk after successful database edit
|
||||
photo.image.save(photo_name, photo_file)
|
||||
except IntegrityError:
|
||||
already_exists += 1
|
||||
continue
|
||||
def save_file():
|
||||
photo.image.save(photo_name, photo_file)
|
||||
|
||||
transaction.on_commit(save_file)
|
||||
|
||||
uploaded_photo_name.append(photo_file.name)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue