Diplomarbeit-Absolventenverein/oqtane.framework
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2492 from sbwalker/dev

Browse Source 
move UI logic from FileService to FileManager, add progressive retry logic, update file attributes if uploading a new version of a file, clean up temporary artifacts on failure, improve upload efficiency
This commit is contained in:
Shaun Walker 2022-11-09 21:11:42 -05:00 committed by GitHub
commit 26e628e189
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 122 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
Oqtane.Client/Modules/Controls/FileManager.razor
View File

@ -1,4 +1,5 @@
@namespace Oqtane.Modules.Controls @namespace Oqtane.Modules.Controls
@using System.Threading
@inherits ModuleControlBase @inherits ModuleControlBase
@inject IFolderService FolderService @inject IFolderService FolderService
@inject IFileService FileService @inject IFileService FileService
@ -53,7 +54,7 @@
} }
</div> </div>
<div class="col mt-2 text-end"> <div class="col mt-2 text-end">
<button type="button" class="btn btn-success" @onclick="UploadFile">@SharedLocalizer["Upload"]</button> <button type="button" class="btn btn-success" @onclick="UploadFiles">@SharedLocalizer["Upload"]</button>
@if (ShowFiles && GetFileId() != -1) @if (ShowFiles && GetFileId() != -1)
{ {
<button type="button" class="btn btn-danger mx-1" @onclick="DeleteFile">@SharedLocalizer["Delete"]</button> <button type="button" class="btn btn-danger mx-1" @onclick="DeleteFile">@SharedLocalizer["Delete"]</button>
@ -304,17 +305,17 @@
} }
} }
private async Task UploadFile() private async Task UploadFiles()
{ {
_message = string.Empty; _message = string.Empty;
var interop = new Interop(JSRuntime); var interop = new Interop(JSRuntime);
var upload = await interop.GetFiles(_fileinputid); var uploads = await interop.GetFiles(_fileinputid);
if (upload.Length > 0) if (uploads.Length > 0)
{ {
string restricted = ""; string restricted = "";
foreach (var file in upload) foreach (var upload in uploads)
{ {
var extension = (file.LastIndexOf(".") != -1) ? file.Substring(file.LastIndexOf(".") + 1) : ""; var extension = (upload.LastIndexOf(".") != -1) ? upload.Substring(upload.LastIndexOf(".") + 1) : "";
if (!Constants.UploadableFiles.Split(',').Contains(extension.ToLower())) if (!Constants.UploadableFiles.Split(',').Contains(extension.ToLower()))
{ {
restricted += (restricted == "" ? "" : ",") + extension; restricted += (restricted == "" ? "" : ",") + extension;
@ -324,48 +325,68 @@
{ {
try try
{ {
string result; // upload the files
if (Folder == Constants.PackagesFolder) var posturl = Utilities.TenantUrl(PageState.Alias, "/api/file/upload");
var folder = (Folder == Constants.PackagesFolder) ? Folder : FolderId.ToString();
await interop.UploadFiles(posturl, folder, _guid, SiteState.AntiForgeryToken);
// uploading is asynchronous so we need to wait for the uploads to complete
// note that this will only wait a maximum of 15 seconds which may not be long enough for very large file uploads
bool success = false;
int attempts = 0;
while (attempts < 5 && !success)
{ {
result = await FileService.UploadFilesAsync(Folder, upload, _guid); attempts += 1;
} Thread.Sleep(1000 * attempts); // progressive retry
else
{ success = true;
result = await FileService.UploadFilesAsync(FolderId, upload, _guid); List<File> files = await FileService.GetFilesAsync(folder);
if (files.Count > 0)
{
foreach (string upload in uploads)
{
if (!files.Exists(item => item.Name == upload))
{
success = false;
}
}
}
} }
if (result == string.Empty) // reset progress indicators
await interop.SetElementAttribute(_guid + "ProgressInfo", "style", "display: none;");
await interop.SetElementAttribute(_guid + "ProgressBar", "style", "display: none;");
if (success)
{ {
await logger.LogInformation("File Upload Succeeded {Files}", upload); await logger.LogInformation("File Upload Succeeded {Files}", uploads);
if (ShowSuccess) if (ShowSuccess)
{ {
_message = Localizer["Success.File.Upload"]; _message = Localizer["Success.File.Upload"];
_messagetype = MessageType.Success; _messagetype = MessageType.Success;
} }
// set FileId to first file in upload collection
await GetFiles();
var file = _files.Where(item => item.Name == upload[0]).FirstOrDefault();
if (file != null)
{
FileId = file.FileId;
await SetImage();
await OnUpload.InvokeAsync(FileId);
}
StateHasChanged();
} }
else else
{ {
await logger.LogError("File Upload Failed For {Files}", result.Replace(",", ", ")); await logger.LogInformation("File Upload Failed Or Is Still In Progress {Files}", uploads);
_message = Localizer["Error.File.Upload"]; _message = Localizer["Error.File.Upload"];
_messagetype = MessageType.Error; _messagetype = MessageType.Error;
} }
// set FileId to first file in upload collection
await GetFiles();
var file = _files.Where(item => item.Name == uploads[0]).FirstOrDefault();
if (file != null)
{
FileId = file.FileId;
await SetImage();
await OnUpload.InvokeAsync(FileId);
}
StateHasChanged();
} }
catch (Exception ex) catch (Exception ex)
{ {
await logger.LogError(ex, "File Upload Failed {Error}", ex.Message); await logger.LogError(ex, "File Upload Failed {Error}", ex.Message);
_message = Localizer["Error.File.Upload"]; _message = Localizer["Error.File.Upload"];
_messagetype = MessageType.Error; _messagetype = MessageType.Error;
} }

2
Oqtane.Client/Resources/Modules/Controls/FileManager.resx
View File

@ -127,7 +127,7 @@
<value>Error Loading Files</value> <value>Error Loading Files</value>
</data> </data>
<data name="Error.File.Upload" xml:space="preserve"> <data name="Error.File.Upload" xml:space="preserve">
<value>File Upload Failed</value> <value>File Upload Failed Or Is Still In Progress</value>
</data> </data>
<data name="Message.File.NotSelected" xml:space="preserve"> <data name="Message.File.NotSelected" xml:space="preserve">
<value>You Have Not Selected A File To Upload</value> <value>You Have Not Selected A File To Upload</value>

60
Oqtane.Client/Services/FileService.cs
View File

@ -2,27 +2,17 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.JSInterop;
using Oqtane.Documentation; using Oqtane.Documentation;
using Oqtane.Models; using Oqtane.Models;
using Oqtane.Shared; using Oqtane.Shared;
using Oqtane.UI;
namespace Oqtane.Services namespace Oqtane.Services
{ {
[PrivateApi("Don't show in the documentation, as everything should use the Interface")] [PrivateApi("Don't show in the documentation, as everything should use the Interface")]
public class FileService : ServiceBase, IFileService public class FileService : ServiceBase, IFileService
{ {
private readonly SiteState _siteState; public FileService(HttpClient http, SiteState siteState) : base(http, siteState) { }
private readonly IJSRuntime _jsRuntime;
public FileService(HttpClient http, SiteState siteState, IJSRuntime jsRuntime) : base(http, siteState)
{
_siteState = siteState;
_jsRuntime = jsRuntime;
}
private string Apiurl => CreateApiUrl("File"); private string Apiurl => CreateApiUrl("File");
@ -75,54 +65,6 @@ namespace Oqtane.Services
return await GetJsonAsync<File>($"{Apiurl}/upload?url={WebUtility.UrlEncode(url)}&folderid={folderId}&name={name}"); return await GetJsonAsync<File>($"{Apiurl}/upload?url={WebUtility.UrlEncode(url)}&folderid={folderId}&name={name}");
} }
public async Task<string> UploadFilesAsync(int folderId, string[] files, string id)
{
return await UploadFilesAsync(folderId.ToString(), files, id);
}
public async Task<string> UploadFilesAsync(string folder, string[] files, string id)
{
string result = "";
var interop = new Interop(_jsRuntime);
await interop.UploadFiles($"{Apiurl}/upload", folder, id, _siteState.AntiForgeryToken);
// uploading files is asynchronous so we need to wait for the upload to complete
bool success = false;
int attempts = 0;
while (attempts < 5 && success == false)
{
Thread.Sleep(2000); // wait 2 seconds
result = "";
List<File> fileList = await GetFilesAsync(folder);
if (fileList.Count > 0)
{
success = true;
foreach (string file in files)
{
if (!fileList.Exists(item => item.Name == file))
{
success = false;
result += file + ",";
}
}
}
attempts += 1;
}
await interop.SetElementAttribute(id + "ProgressInfo", "style", "display: none;");
await interop.SetElementAttribute(id + "ProgressBar", "style", "display: none;");
if (!success)
{
result = result.Substring(0, result.Length - 1);
}
return result;
}
public async Task<byte[]> DownloadFileAsync(int fileId) public async Task<byte[]> DownloadFileAsync(int fileId)
{ {
return await GetByteArrayAsync($"{Apiurl}/download/{fileId}"); return await GetByteArrayAsync($"{Apiurl}/download/{fileId}");

21
Oqtane.Client/Services/Interfaces/IFileService.cs
View File

@ -66,27 +66,6 @@ namespace Oqtane.Services
/// <returns></returns> /// <returns></returns>
Task<File> UploadFileAsync(string url, int folderId, string name); Task<File> UploadFileAsync(string url, int folderId, string name);
/// <summary>
/// Upload one or more files.
/// </summary>
/// <param name="folderId">Target <see cref="Folder"/></param>
/// <param name="files">The files to upload, serialized as a string.</param>
/// <param name="fileUploadName">A task-identifier, to ensure communication about this upload.</param>
/// <returns></returns>
Task<string> UploadFilesAsync(int folderId, string[] files, string fileUploadName);
/// <summary>
/// Upload one or more files.
/// </summary>
/// <param name="folder">Target <see cref="Folder"/>
/// TODO: todoc verify exactly from where the folder path must start
/// </param>
/// <param name="files">The files to upload, serialized as a string.</param>
/// <param name="fileUploadName">A task-identifier, to ensure communication about this upload.</param>
/// <returns></returns>
Task<string> UploadFilesAsync(string folder, string[] files, string fileUploadName);
/// <summary> /// <summary>
/// Get / download a file (the body). /// Get / download a file (the body).
/// </summary> /// </summary>

94
Oqtane.Server/Controllers/FileController.cs
View File

@ -339,7 +339,15 @@ namespace Oqtane.Controllers
var file = CreateFile(upload, FolderId, Path.Combine(folderPath, upload)); var file = CreateFile(upload, FolderId, Path.Combine(folderPath, upload));
if (file != null) if (file != null)
{ {
file = _files.AddFile(file); if (file.FileId == 0)
{
file = _files.AddFile(file);
}
else
{
file = _files.UpdateFile(file);
}
_logger.Log(LogLevel.Information, this, LogFunction.Create, "File Upload Succeeded {File}", Path.Combine(folderPath, upload));
_syncManager.AddSyncEvent(_alias.TenantId, EntityNames.File, file.FileId, SyncEventActions.Create); _syncManager.AddSyncEvent(_alias.TenantId, EntityNames.File, file.FileId, SyncEventActions.Create);
} }
} }
@ -361,16 +369,16 @@ namespace Oqtane.Controllers
int totalparts = int.Parse(parts?.Substring(parts.IndexOf("_") + 1)); int totalparts = int.Parse(parts?.Substring(parts.IndexOf("_") + 1));
filename = Path.GetFileNameWithoutExtension(filename); // base filename filename = Path.GetFileNameWithoutExtension(filename); // base filename
string[] fileParts = Directory.GetFiles(folder, filename + token + "*"); // list of all file parts string[] fileparts = Directory.GetFiles(folder, filename + token + "*"); // list of all file parts
// if all of the file parts exist ( note that file parts can arrive out of order ) // if all of the file parts exist ( note that file parts can arrive out of order )
if (fileParts.Length == totalparts && CanAccessFiles(fileParts)) if (fileparts.Length == totalparts && CanAccessFiles(fileparts))
{ {
// merge file parts // merge file parts into temp file ( in case another user is trying to get the file )
bool success = true; bool success = true;
using (var stream = new FileStream(Path.Combine(folder, filename + ".tmp"), FileMode.Create)) using (var stream = new FileStream(Path.Combine(folder, filename + ".tmp"), FileMode.Create))
{ {
foreach (string filepart in fileParts) foreach (string filepart in fileparts)
{ {
try try
{ {
@ -386,52 +394,49 @@ namespace Oqtane.Controllers
} }
} }
// delete file parts and rename file // clean up file parts
foreach (var file in Directory.GetFiles(folder, "*" + token + "*"))
{
// file name matches part or is more than 2 hours old (ie. a prior file upload failed)
if (fileparts.Contains(file) || System.IO.File.GetCreationTime(file).ToUniversalTime() < DateTime.UtcNow.AddHours(-2))
{
System.IO.File.Delete(file);
}
}
// rename temp file
if (success) if (success)
{ {
foreach (string filepart in fileParts) // remove file if it already exists (as well as any thumbnails)
foreach (var file in Directory.GetFiles(folder, Path.GetFileNameWithoutExtension(filename) + ".*"))
{ {
System.IO.File.Delete(filepart); if (Path.GetExtension(file) != ".tmp")
{
System.IO.File.Delete(file);
}
} }
// remove file if it already exists // rename temp file now that the entire process is completed
if (System.IO.File.Exists(Path.Combine(folder, filename)))
{
System.IO.File.Delete(Path.Combine(folder, filename));
}
// rename file now that the entire process is completed
System.IO.File.Move(Path.Combine(folder, filename + ".tmp"), Path.Combine(folder, filename)); System.IO.File.Move(Path.Combine(folder, filename + ".tmp"), Path.Combine(folder, filename));
_logger.Log(LogLevel.Information, this, LogFunction.Create, "File Uploaded {File}", Path.Combine(folder, filename));
// return filename
merged = filename; merged = filename;
} }
} }
// clean up file parts which are more than 2 hours old ( which can happen if a prior file upload failed )
var cleanupFiles = Directory.EnumerateFiles(folder, "*" + token + "*")
.Where(f => Path.GetExtension(f).StartsWith(token) && !Path.GetFileName(f).StartsWith(filename));
foreach (var file in cleanupFiles)
{
var createdDate = System.IO.File.GetCreationTime(file).ToUniversalTime();
if (createdDate < DateTime.UtcNow.AddHours(-2))
{
System.IO.File.Delete(file);
}
}
return merged; return merged;
} }
private bool CanAccessFiles(string[] files) private bool CanAccessFiles(string[] files)
{ {
// ensure files are not locked by another process ( ie. still being written to ) // ensure files are not locked by another process
bool canaccess = true;
FileStream stream = null; FileStream stream = null;
bool locked = false;
foreach (string file in files) foreach (string file in files)
{ {
locked = true;
int attempts = 0; int attempts = 0;
bool locked = true; // note that this will wait a maximum of 15 seconds for a file to become unlocked
while (attempts < 5 && locked) while (attempts < 5 && locked)
{ {
try try
@ -441,7 +446,8 @@ namespace Oqtane.Controllers
} }
catch // file is locked by another process catch // file is locked by another process
{ {
Thread.Sleep(1000); // wait 1 second attempts += 1;
Thread.Sleep(1000 * attempts); // progressive retry
} }
finally finally
{ {
@ -450,20 +456,15 @@ namespace Oqtane.Controllers
stream.Close(); stream.Close();
} }
} }
attempts += 1;
} }
if (locked)
if (locked && canaccess)
{ {
canaccess = false; break; // file still locked after retrying
} }
} }
return !locked;
return canaccess;
} }
/// <summary> /// <summary>
/// Get file with header /// Get file with header
/// Content-Disposition: inline /// Content-Disposition: inline
@ -655,7 +656,7 @@ namespace Oqtane.Controllers
private Models.File CreateFile(string filename, int folderid, string filepath) private Models.File CreateFile(string filename, int folderid, string filepath)
{ {
Models.File file = null; var file = _files.GetFile(folderid, filename);
int size = 0; int size = 0;
var folder = _folders.GetFolder(folderid); var folder = _folders.GetFolder(folderid);
@ -670,7 +671,10 @@ namespace Oqtane.Controllers
FileInfo fileinfo = new FileInfo(filepath); FileInfo fileinfo = new FileInfo(filepath);
if (folder.Capacity == 0 || ((size + fileinfo.Length) / 1000000) < folder.Capacity) if (folder.Capacity == 0 || ((size + fileinfo.Length) / 1000000) < folder.Capacity)
{ {
file = new Models.File(); if (file == null)
{
file = new Models.File();
}
file.Name = filename; file.Name = filename;
file.FolderId = folderid; file.FolderId = folderid;
@ -700,7 +704,11 @@ namespace Oqtane.Controllers
else else
{ {
System.IO.File.Delete(filepath); System.IO.File.Delete(filepath);
_logger.Log(LogLevel.Warning, this, LogFunction.Create, "File Exceeds Folder Capacity {Folder} {File}", folder, filepath); if (file != null)
{
_files.DeleteFile(file.FileId);
}
_logger.Log(LogLevel.Warning, this, LogFunction.Create, "File Exceeds Folder Capacity And Has Been Removed {Folder} {File}", folder, filepath);
} }
return file; return file;

17
Oqtane.Server/Repository/FileRepository.cs
View File

@ -83,6 +83,23 @@ namespace Oqtane.Repository
return file; return file;
} }
public File GetFile(int folderId, string fileName)
{
var file = _db.File.AsNoTracking()
.Include(item => item.Folder)
.FirstOrDefault(item => item.FolderId == folderId &&
item.Name.ToLower() == fileName);
if (file != null)
{
IEnumerable<Permission> permissions = _permissions.GetPermissions(file.Folder.SiteId, EntityNames.Folder, file.FolderId).ToList();
file.Folder.Permissions = permissions.EncodePermissions();
file.Url = GetFileUrl(file, _tenants.GetAlias());
}
return file;
}
public File GetFile(int siteId, string folderPath, string fileName) public File GetFile(int siteId, string folderPath, string fileName)
{ {
var file = _db.File.AsNoTracking() var file = _db.File.AsNoTracking()

1
Oqtane.Server/Repository/Interfaces/IFileRepository.cs
View File

@ -10,6 +10,7 @@ namespace Oqtane.Repository
File UpdateFile(File file); File UpdateFile(File file);
File GetFile(int fileId); File GetFile(int fileId);
File GetFile(int fileId, bool tracking); File GetFile(int fileId, bool tracking);
File GetFile(int folderId, string fileName);
File GetFile(int siteId, string folderPath, string fileName); File GetFile(int siteId, string folderPath, string fileName);
void DeleteFile(int fileId); void DeleteFile(int fileId);
string GetFilePath(int fileId); string GetFilePath(int fileId);

2
Oqtane.Shared/Shared/Constants.cs
View File

@ -46,7 +46,7 @@ namespace Oqtane.Shared
public const string DefaultSite = "Default Site"; public const string DefaultSite = "Default Site";
public const string ImageFiles = "jpg,jpeg,jpe,gif,bmp,png,ico,webp"; public const string ImageFiles = "jpg,jpeg,jpe,gif,bmp,png,ico,webp";
public const string UploadableFiles = ImageFiles + ",mov,wmv,avi,mp4,mp3,doc,docx,xls,xlsx,ppt,pptx,pdf,txt,zip,nupkg,csv,json,xml,xslt,rss,html,htm,css"; public const string UploadableFiles = ImageFiles + ",mov,wmv,avi,mp4,mp3,doc,docx,xls,xlsx,ppt,pptx,pdf,txt,zip,nupkg,csv,json,xml,rss,css";
public const string ReservedDevices = "CON,NUL,PRN,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9,CONIN$,CONOUT$"; public const string ReservedDevices = "CON,NUL,PRN,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9,CONIN$,CONOUT$";
public static readonly char[] InvalidFileNameChars = public static readonly char[] InvalidFileNameChars =