fix: 5058 - chunk file upload concurrency limit and customizable site settings

Oqtane.Client/Modules/Admin/Site/Index.razor

@@ -156,6 +156,18 @@
                         <input id="uploadableFileExt" spellcheck="false" class="form-control" @bind="@_uploadableFiles" />
                     </div>
                 </div>
+                <div class="row mb-1 align-items-center">
+                    <Label Class="col-sm-3" For="maxChunkSizeMB" HelpText="Enter the maximum size in MB of a chunk for file uploads" ResourceKey="MaxChunkSize">Maximum File Chunk Size (MB): </Label>
+                    <div class="col-sm-9">
+                        <input id="maxChunkSizeMB" type="number" min="1" max="50" step="1" class="form-control" @bind="@_maxChunkSizeMB" />
+                    </div>
+                </div>
+                <div class="row mb-1 align-items-center">
+                    <Label Class="col-sm-3" For="maxConcurrentChunkUploads" HelpText="Enter the maximum concurrent number of file chunk uploads. If set to 0, no concurrency limit is applied" ResourceKey="MaxConcurrentChunkUploads">Maximum Concurrent File Chunk Uploads: </Label>
+                    <div class="col-sm-9">
+                        <input id="maxConcurrentChunkUploads" type="number" min="0" step="1" class="form-control" @bind="@_maxConcurrentChunkUploads" />
+                    </div>
+                </div>
             </div>
         </Section>
         <Section Name="PageContent" Heading="Page Content" ResourceKey="PageContent">
@@ -432,6 +444,8 @@
     private string _textEditor = "";
     private string _imageFiles = string.Empty;
     private string _uploadableFiles = string.Empty;
+    private int _maxChunkSizeMB = 1;
+    private int _maxConcurrentChunkUploads = 0;
 
     private string _headcontent = string.Empty;
     private string _bodycontent = string.Empty;
@@ -530,6 +544,8 @@
                 _imageFiles = (string.IsNullOrEmpty(_imageFiles)) ? Constants.ImageFiles : _imageFiles;
                 _uploadableFiles = SettingService.GetSetting(settings, "UploadableFiles", Constants.UploadableFiles);
                 _uploadableFiles = (string.IsNullOrEmpty(_uploadableFiles)) ? Constants.UploadableFiles : _uploadableFiles;
+                _maxChunkSizeMB = int.Parse(SettingService.GetSetting(settings, "MaxChunkSizeMB", "1"));
+                _maxConcurrentChunkUploads = int.Parse(SettingService.GetSetting(settings, "MaxConcurrentChunkUploads", "0"));
 
                 // page content
                 _headcontent = site.HeadContent;
@@ -736,6 +752,8 @@
                         settings = SettingService.SetSetting(settings, "TextEditor", _textEditor);
                         settings = SettingService.SetSetting(settings, "ImageFiles", (_imageFiles != Constants.ImageFiles) ? _imageFiles.Replace(" ", "") : "", false);
                         settings = SettingService.SetSetting(settings, "UploadableFiles", (_uploadableFiles != Constants.UploadableFiles) ? _uploadableFiles.Replace(" ", "") : "", false);
+                        settings = SettingService.SetSetting(settings, "MaxChunkSizeMB", _maxChunkSizeMB.ToString(), false);
+                        settings = SettingService.SetSetting(settings, "MaxConcurrentChunkUploads", _maxConcurrentChunkUploads.ToString(), false);
 
                         await SettingService.UpdateSiteSettingsAsync(settings, site.SiteId);
 

Oqtane.Client/Modules/Controls/FileManager.razor

@@ -121,6 +121,9 @@
     private MessageType _messagetype;
     private bool _uploading = false;
 
+    private int _maxChunkSizeMB = 1;
+    private int _maxConcurrentUploads = 0;
+
     [Parameter]
     public string Id { get; set; } // optional - for setting the id of the FileManager component for accessibility
 
@@ -173,6 +176,9 @@
         _fileinputid = "FileInput_" + _guid;
         _progressinfoid = "ProgressInfo_" + _guid;
         _progressbarid = "ProgressBar_" + _guid;
+
+        int.TryParse(SettingService.GetSetting(PageState.Site.Settings, "MaxChunkSizeMB", "1"), out _maxChunkSizeMB);
+        int.TryParse(SettingService.GetSetting(PageState.Site.Settings, "MaxConcurrentChunkUploads", "0"), out _maxConcurrentUploads);
     }
 
     protected override async Task OnParametersSetAsync()
@@ -383,7 +389,7 @@
                         StateHasChanged();
                     }
 
-                    await interop.UploadFiles(posturl, folder, _guid, SiteState.AntiForgeryToken, jwt);
+                    await interop.UploadFiles(posturl, folder, _guid, SiteState.AntiForgeryToken, jwt, _maxChunkSizeMB, _maxConcurrentUploads);
 
                     // uploading is asynchronous so we need to poll to determine if uploads are completed
                     var success = true;

Oqtane.Client/Resources/Modules/Admin/Site/Index.resx

@@ -438,4 +438,16 @@
   <data name="System" xml:space="preserve">
     <value>System</value>
   </data>
+  <data name="MaxChunkSize.HelpText" xml:space="preserve">
+    <value>Enter the maximum size in MB of a chunk for file uploads</value>
+  </data>
+  <data name="MaxChunkSize.Text" xml:space="preserve">
+    <value>Maximum File Chunk Size (MB):</value>
+  </data>
+  <data name="MaxConcurrentChunkUploads.HelpText" xml:space="preserve">
+    <value>Enter the maximum concurrent number of file chunk uploads. If set to 0, no concurrency limit is applied</value>
+  </data>
+  <data name="MaxConcurrentChunkUploads.Text" xml:space="preserve">
+    <value>Maximum Concurrent File Chunk Uploads:</value>
+  </data>
 </root>

Oqtane.Client/UI/Interop.cs

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
 using System.Text.Json;
 using System.Collections.Generic;
 using System.Linq;
+using System;
 
 namespace Oqtane.UI
 {
@@ -208,13 +209,19 @@ namespace Oqtane.UI
             }
         }
 
+        [Obsolete("This function is deprecated. Use UploadFiles with MaxChunkSize and MaxConcurrentUploads parameters instead.", false)]
         public Task UploadFiles(string posturl, string folder, string id, string antiforgerytoken, string jwt)
+        {
+            return UploadFiles(posturl, folder, id, antiforgerytoken, jwt, 1, 0);
+        }
+
+        public Task UploadFiles(string posturl, string folder, string id, string antiforgerytoken, string jwt, int maxChunkSizeMB, int maxConcurrentUploads)
         {
             try
             {
                 _jsRuntime.InvokeVoidAsync(
                     "Oqtane.Interop.uploadFiles",
-                    posturl, folder, id, antiforgerytoken, jwt);
+                    posturl, folder, id, antiforgerytoken, jwt, maxChunkSizeMB, maxConcurrentUploads);
                 return Task.CompletedTask;
             }
             catch

Oqtane.Server/wwwroot/js/interop.js

@@ -308,7 +308,7 @@ Oqtane.Interop = {
         }
         return files;
     },
-    uploadFiles: function (posturl, folder, id, antiforgerytoken, jwt) {
+    uploadFiles: function (posturl, folder, id, antiforgerytoken, jwt, maxChunkSizeMB, maxConcurrentUploads) {
         var fileinput = document.getElementById('FileInput_' + id);
         var progressinfo = document.getElementById('ProgressInfo_' + id);
         var progressbar = document.getElementById('ProgressBar_' + id);
@@ -326,10 +326,22 @@ Oqtane.Interop = {
             totalSize = totalSize + files[i].size;
         }
 
-        var maxChunkSizeMB = 1;
+        maxChunkSizeMB = Math.ceil(maxChunkSizeMB);
+        if (maxChunkSizeMB < 1) {
+            maxChunkSizeMB = 1;
+        }
+        else if (maxChunkSizeMB > 50) {
+            maxChunkSizeMB = 50;
+        }
+
         var bufferChunkSize = maxChunkSizeMB * (1024 * 1024);
         var uploadedSize = 0;
 
+        maxConcurrentUploads = Math.ceil(maxConcurrentUploads);
+        var hasConcurrencyLimit = maxConcurrentUploads > 0;
+        var uploadQueue = [];
+        var activeUploads = 0;
+
         for (var i = 0; i < files.length; i++) {
             var fileChunk = [];
             var file = files[i];
@@ -376,13 +388,23 @@ Oqtane.Interop = {
                     }
                 };
                 request.upload.onloadend = function (e) {
+                    if (hasConcurrencyLimit) {
+                        activeUploads--;
+                        processUploads();
+                    }
+
                     if (progressinfo !== null && progressbar !== null) {
                         uploadedSize = uploadedSize + e.total;
                         var percent = Math.ceil((uploadedSize / totalSize) * 100);
                         progressbar.value = (percent / 100);
                     }
                 };
-                request.upload.onerror = function() {
+                request.upload.onerror = function () {
+                    if (hasConcurrencyLimit) {
+                        activeUploads--;
+                        processUploads();
+                    }
+
                     if (progressinfo !== null && progressbar !== null) {
                         if (files.length === 1) {
                             progressinfo.innerHTML = file.name + ' Error: ' + request.statusText;
@@ -392,13 +414,33 @@ Oqtane.Interop = {
                         }
                     }
                 };
-                request.send(data);
+
+                if (hasConcurrencyLimit) {
+                    uploadQueue.push({ data, request });
+                    processUploads();
+                }
+                else {
+                    request.send(data);
+                }
             }
 
             if (i === files.length - 1) {
                 fileinput.value = '';
             }
         }
+
+        function processUploads() {
+            if (uploadQueue.length === 0 || activeUploads >= maxConcurrentUploads) {
+                return;
+            }
+
+            while (activeUploads < maxConcurrentUploads && uploadQueue.length > 0) {
+                activeUploads++;
+
+                let { data, request } = uploadQueue.shift();
+                request.send(data);
+            }
+        }
     },
     refreshBrowser: function (verify, wait) {
         async function attemptReload (verify) {