From a84eee8782c4d7160f58521f0872f4e98b081a82 Mon Sep 17 00:00:00 2001 From: Shaun Walker Date: Fri, 6 Sep 2019 13:15:18 -0400 Subject: [PATCH] Large file streaming uploads --- .gitignore | 4 +- .../Modules/Admin/ModuleDefinitions/Add.razor | 30 ++++++ .../Admin/ModuleDefinitions/Index.razor | 1 + .../Modules/Controls/FileUpload.razor | 43 ++++++++ .../Modules/Controls/PermissionGrid.razor | 4 +- Oqtane.Client/Modules/IModuleControl.cs | 2 +- Oqtane.Client/Modules/ModuleBase.cs | 2 +- Oqtane.Client/Services/FileService.cs | 37 +++++++ .../Services/Interfaces/IFileService.cs | 12 +++ Oqtane.Client/Shared/Container.razor | 8 +- Oqtane.Client/Shared/Interop.cs | 15 +++ Oqtane.Client/Shared/SiteRouter.razor | 2 +- Oqtane.Client/Startup.cs | 1 + Oqtane.Server/Controllers/FileController.cs | 98 +++++++++++++++++++ Oqtane.Server/Startup.cs | 1 + Oqtane.Server/wwwroot/js/interop.js | 56 +++++++++++ Oqtane.Shared/Models/Module.cs | 2 +- 17 files changed, 305 insertions(+), 13 deletions(-) create mode 100644 Oqtane.Client/Modules/Admin/ModuleDefinitions/Add.razor create mode 100644 Oqtane.Client/Modules/Controls/FileUpload.razor create mode 100644 Oqtane.Client/Services/FileService.cs create mode 100644 Oqtane.Client/Services/Interfaces/IFileService.cs create mode 100644 Oqtane.Server/Controllers/FileController.cs diff --git a/.gitignore b/.gitignore index 251d9eeb..33c751ad 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,9 @@ artifacts/ msbuild.binlog .vscode/ *.binlog +*.nupkg Oqtane.Server/appsettings.json Oqtane.Server/Data/*.mdf -Oqtane.Server/Data/*.ldf \ No newline at end of file +Oqtane.Server/Data/*.ldf + diff --git a/Oqtane.Client/Modules/Admin/ModuleDefinitions/Add.razor b/Oqtane.Client/Modules/Admin/ModuleDefinitions/Add.razor new file mode 100644 index 00000000..5c8b86a4 --- /dev/null +++ b/Oqtane.Client/Modules/Admin/ModuleDefinitions/Add.razor @@ -0,0 +1,30 @@ +@using Microsoft.AspNetCore.Components.Routing +@using Oqtane.Client.Modules.Controls +@using Oqtane.Modules +@using Oqtane.Services +@inherits ModuleBase +@inject IUriHelper UriHelper +@inject IFileService FileService + + + + + + +
+ + + +
+ +Cancel + +@code { + public override SecurityAccessLevel SecurityAccessLevel { get { return SecurityAccessLevel.Host; } } + + private async Task UploadFile() + { + await FileService.UploadFilesAsync("/Sites/Modules"); + } + +} diff --git a/Oqtane.Client/Modules/Admin/ModuleDefinitions/Index.razor b/Oqtane.Client/Modules/Admin/ModuleDefinitions/Index.razor index 87c2beff..50fefe30 100644 --- a/Oqtane.Client/Modules/Admin/ModuleDefinitions/Index.razor +++ b/Oqtane.Client/Modules/Admin/ModuleDefinitions/Index.razor @@ -12,6 +12,7 @@ } else { + diff --git a/Oqtane.Client/Modules/Controls/FileUpload.razor b/Oqtane.Client/Modules/Controls/FileUpload.razor new file mode 100644 index 00000000..b93de437 --- /dev/null +++ b/Oqtane.Client/Modules/Controls/FileUpload.razor @@ -0,0 +1,43 @@ +@if (multiple) +{ + +} +else +{ + +} + + +@code { + [Parameter] + public string Name { get; set; } // optional - can be used for managing multiple file upload controls on a page + + [Parameter] + public string Filter { get; set; } // optional - for restricting types of files that can be selected + + [Parameter] + public string Multiple { get; set; } // optional - enable multiple file uploads + + string fileid = ""; + string progressinfoid = ""; + string progressbarid = ""; + string filter = "*"; + bool multiple = false; + + protected override void OnInitialized() + { + fileid = Name + "FileInput"; + progressinfoid = Name + "ProgressInfo"; + progressbarid = Name + "ProgressBar"; + + if (!string.IsNullOrEmpty(Filter)) + { + filter = Filter; + } + + if (!string.IsNullOrEmpty(Multiple)) + { + multiple = bool.Parse(Multiple); + } + } +} diff --git a/Oqtane.Client/Modules/Controls/PermissionGrid.razor b/Oqtane.Client/Modules/Controls/PermissionGrid.razor index e2f40f00..fa1c9774 100644 --- a/Oqtane.Client/Modules/Controls/PermissionGrid.razor +++ b/Oqtane.Client/Modules/Controls/PermissionGrid.razor @@ -67,9 +67,9 @@
- + - +
diff --git a/Oqtane.Client/Modules/IModuleControl.cs b/Oqtane.Client/Modules/IModuleControl.cs index 96e59364..3fd7516c 100644 --- a/Oqtane.Client/Modules/IModuleControl.cs +++ b/Oqtane.Client/Modules/IModuleControl.cs @@ -5,6 +5,6 @@ SecurityAccessLevel SecurityAccessLevel { get; } // defines the security access level for this control - defaults to View string Title { get; } // title to display for this control - defaults to module title string Actions { get; } // allows for routing by configuration rather than by convention ( comma delimited ) - defaults to using component file name - string ContainerType { get; } // container for embedding control - defaults to AdminContainer + bool UseAdminContainer { get; } // container for embedding module control - defaults to true } } diff --git a/Oqtane.Client/Modules/ModuleBase.cs b/Oqtane.Client/Modules/ModuleBase.cs index af18ead7..6f176c38 100644 --- a/Oqtane.Client/Modules/ModuleBase.cs +++ b/Oqtane.Client/Modules/ModuleBase.cs @@ -18,7 +18,7 @@ namespace Oqtane.Modules public virtual string Actions { get { return ""; } } - public virtual string ContainerType { get { return ""; } } + public virtual bool UseAdminContainer { get { return true; } } public string NavigateUrl() { diff --git a/Oqtane.Client/Services/FileService.cs b/Oqtane.Client/Services/FileService.cs new file mode 100644 index 00000000..57eddcc2 --- /dev/null +++ b/Oqtane.Client/Services/FileService.cs @@ -0,0 +1,37 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Components; +using Microsoft.JSInterop; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + public class FileService : ServiceBase, IFileService + { + private readonly SiteState sitestate; + private readonly IUriHelper urihelper; + private readonly IJSRuntime jsRuntime; + + public FileService(SiteState sitestate, IUriHelper urihelper, IJSRuntime jsRuntime) + { + this.sitestate = sitestate; + this.urihelper = urihelper; + this.jsRuntime = jsRuntime; + } + + private string apiurl + { + get { return CreateApiUrl(sitestate.Alias, urihelper.GetAbsoluteUri(), "File"); } + } + + public async Task UploadFilesAsync(string Folder) + { + await UploadFilesAsync(Folder, ""); + } + + public async Task UploadFilesAsync(string Folder, string FileUploadName) + { + var interop = new Interop(jsRuntime); + await interop.UploadFiles(apiurl + "/upload", Folder, FileUploadName); + } + } +} diff --git a/Oqtane.Client/Services/Interfaces/IFileService.cs b/Oqtane.Client/Services/Interfaces/IFileService.cs new file mode 100644 index 00000000..c0943d35 --- /dev/null +++ b/Oqtane.Client/Services/Interfaces/IFileService.cs @@ -0,0 +1,12 @@ +using Oqtane.Models; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Oqtane.Services +{ + public interface IFileService + { + Task UploadFilesAsync(string Folder); + Task UploadFilesAsync(string Folder, string FileUploadName); + } +} diff --git a/Oqtane.Client/Shared/Container.razor b/Oqtane.Client/Shared/Container.razor index d19019d5..8bbec6df 100644 --- a/Oqtane.Client/Shared/Container.razor +++ b/Oqtane.Client/Shared/Container.razor @@ -47,13 +47,9 @@ { ModuleState = Module; // passed in from Pane component container = ModuleState.ContainerType; - if (PageState.ModuleId != -1 && PageState.Control != "") + if (PageState.ModuleId != -1 && PageState.Control != "" && ModuleState.UseAdminContainer) { - container = ModuleState.AdminContainerType; - if (container == "") - { - container = Constants.DefaultAdminContainer; - } + container = Constants.DefaultAdminContainer; } } return Task.CompletedTask; diff --git a/Oqtane.Client/Shared/Interop.cs b/Oqtane.Client/Shared/Interop.cs index 3ece9f1d..86430083 100644 --- a/Oqtane.Client/Shared/Interop.cs +++ b/Oqtane.Client/Shared/Interop.cs @@ -85,5 +85,20 @@ namespace Oqtane.Shared return Task.CompletedTask; } } + + public Task UploadFiles(string posturl, string folder, string name) + { + try + { + jsRuntime.InvokeAsync( + "interop.uploadFiles", + posturl, folder, name); + return Task.CompletedTask; + } + catch + { + return Task.CompletedTask; + } + } } } diff --git a/Oqtane.Client/Shared/SiteRouter.razor b/Oqtane.Client/Shared/SiteRouter.razor index 4a4e29fa..74e23b2b 100644 --- a/Oqtane.Client/Shared/SiteRouter.razor +++ b/Oqtane.Client/Shared/SiteRouter.razor @@ -349,7 +349,7 @@ module.SecurityAccessLevel = (SecurityAccessLevel)moduletype.GetProperty("SecurityAccessLevel").GetValue(moduleobject, null); module.ControlTitle = (string)moduletype.GetProperty("Title").GetValue(moduleobject); module.Actions = (string)moduletype.GetProperty("Actions").GetValue(moduleobject); - module.AdminContainerType = (string)moduletype.GetProperty("ContainerType").GetValue(moduleobject); + module.UseAdminContainer = (bool)moduletype.GetProperty("UseAdminContainer").GetValue(moduleobject); } } diff --git a/Oqtane.Client/Startup.cs b/Oqtane.Client/Startup.cs index 2b8f456b..fbb94e81 100644 --- a/Oqtane.Client/Startup.cs +++ b/Oqtane.Client/Startup.cs @@ -50,6 +50,7 @@ namespace Oqtane.Client services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); // dynamically register module contexts and repository services Assembly[] assemblies = AppDomain.CurrentDomain.GetAssemblies(); diff --git a/Oqtane.Server/Controllers/FileController.cs b/Oqtane.Server/Controllers/FileController.cs new file mode 100644 index 00000000..430b1cbf --- /dev/null +++ b/Oqtane.Server/Controllers/FileController.cs @@ -0,0 +1,98 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Oqtane.Controllers +{ + [Route("{site}/api/[controller]")] + public class FileController : Controller + { + private readonly IWebHostEnvironment environment; + + public FileController(IWebHostEnvironment environment) + { + this.environment = environment; + } + + // GET api//current + [HttpPost("upload")] + public async Task UploadFile(string folder, IFormFile file) + { + if (file.Length > 0) + { + if (!folder.Contains(":\\")) + { + folder = folder.Replace("/", "\\"); + if (folder.StartsWith("\\")) folder = folder.Substring(1); + folder = Path.Combine(environment.WebRootPath, folder); + } + if (!Directory.Exists(folder)) + { + Directory.CreateDirectory(folder); + } + using (var stream = new FileStream(Path.Combine(folder, file.FileName), FileMode.Create)) + { + await file.CopyToAsync(stream); + } + await MergeFile(folder, file.FileName); + } + } + + private async Task MergeFile(string folder, string filename) + { + // parse the filename which is in the format of filename.ext.part_x_y + string token = ".part_"; + string parts = Path.GetExtension(filename).Replace(token, ""); // returns "x_y" + int totalparts = int.Parse(parts.Substring(parts.IndexOf("_") + 1)); + filename = filename.Substring(0, filename.IndexOf(token)); // base filename + string[] fileparts = Directory.GetFiles(folder, filename + token + "*"); // list of all file parts + + // if all of the file parts exist ( note that file parts can arrive out of order ) + if (fileparts.Length == totalparts) + { + // merge file parts + bool success = true; + using (var stream = new FileStream(Path.Combine(folder, filename), FileMode.Create)) + { + foreach (string filepart in fileparts) + { + try + { + using (FileStream chunk = new FileStream(filepart, FileMode.Open)) + { + await chunk.CopyToAsync(stream); + } + } + catch + { + success = false; + } + } + } + + // delete file parts + if (success) + { + foreach (string filepart in fileparts) + { + System.IO.File.Delete(filepart); + } + } + } + + // clean up file parts which are more than 2 hours old ( which can happen if a file upload failed ) + fileparts = Directory.GetFiles(folder, "*" + token + "*"); + foreach (string filepart in fileparts) + { + DateTime createddate = System.IO.File.GetCreationTime(filepart); + if (createddate < DateTime.Now.AddHours(-2)) + { + System.IO.File.Delete(filepart); + } + } + } + } +} diff --git a/Oqtane.Server/Startup.cs b/Oqtane.Server/Startup.cs index 516e63cb..e8fa612b 100644 --- a/Oqtane.Server/Startup.cs +++ b/Oqtane.Server/Startup.cs @@ -95,6 +95,7 @@ namespace Oqtane.Server services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddSingleton(); diff --git a/Oqtane.Server/wwwroot/js/interop.js b/Oqtane.Server/wwwroot/js/interop.js index db61ecbe..7b23a474 100644 --- a/Oqtane.Server/wwwroot/js/interop.js +++ b/Oqtane.Server/wwwroot/js/interop.js @@ -55,5 +55,61 @@ window.interop = { document.body.appendChild(form); form.submit(); + }, + uploadFiles: function (posturl, folder, name) { + var files = document.getElementById(name + 'FileInput').files; + var progressinfo = document.getElementById(name + 'ProgressInfo'); + var progressbar = document.getElementById(name + 'ProgressBar'); + var filename = ''; + + for (var i = 0; i < files.length; i++) { + var FileChunk = []; + var file = files[i]; + var MaxFileSizeMB = 1; + var BufferChunkSize = MaxFileSizeMB * (1024 * 1024); + var FileStreamPos = 0; + var EndPos = BufferChunkSize; + var Size = file.size; + + progressbar.setAttribute("style", "visibility: visible;"); + + if (files.length > 1) { + filename = file.name; + } + + while (FileStreamPos < Size) { + FileChunk.push(file.slice(FileStreamPos, EndPos)); + FileStreamPos = EndPos; + EndPos = FileStreamPos + BufferChunkSize; + } + + var TotalParts = FileChunk.length; + var PartCount = 0; + + while (Chunk = FileChunk.shift()) { + PartCount++; + var FileName = file.name + ".part_" + PartCount + "_" + TotalParts; + + var data = new FormData(); + data.append('folder', folder); + data.append('file', Chunk, FileName); + var request = new XMLHttpRequest(); + request.open('POST', posturl, true); + request.upload.onloadstart = function (e) { + progressbar.value = 0; + progressinfo.innerHTML = filename + ' 0%'; + }; + request.upload.onprogress = function (e) { + var percent = Math.ceil((e.loaded / e.total) * 100); + progressbar.value = (percent / 100); + progressinfo.innerHTML = filename + '[' + PartCount + '] ' + percent + '%'; + }; + request.upload.onloadend = function (e) { + progressbar.value = 1; + progressinfo.innerHTML = filename + ' 100%'; + }; + request.send(data); + } + } } }; diff --git a/Oqtane.Shared/Models/Module.cs b/Oqtane.Shared/Models/Module.cs index a5687f98..8816205a 100644 --- a/Oqtane.Shared/Models/Module.cs +++ b/Oqtane.Shared/Models/Module.cs @@ -49,6 +49,6 @@ namespace Oqtane.Models [NotMapped] public string Actions { get; set; } [NotMapped] - public string AdminContainerType { get; set; } + public bool UseAdminContainer { get; set; } } }