kocoded/oqtane.framework
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

include purge job for maintaining event logs and visitor logs

Browse Source
This commit is contained in:
Shaun Walker 2022-01-07 23:30:29 -05:00
parent a2f8fe3694
commit e2688e6feb
13 changed files with 458 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Oqtane.Client/Modules/Admin/Logs/Index.razor
View File

@ -1,6 +1,7 @@
@namespace Oqtane.Modules.Admin.Logs
@inherits ModuleBase
@inject ILogService LogService
@inject ISettingService SettingService
@inject IStringLocalizer<Index> Localizer
@inject IStringLocalizer<SharedResources> SharedLocalizer
@ -10,6 +11,8 @@
}
else
{
<TabStrip>
<TabPanel Name="Events" Heading="Events" ResourceKey="Events">
<div class="container g-0">
<div class="row mb-1 align-items-center">
<div class="col-sm-4">
@ -71,6 +74,20 @@ else
{
<p><em>@Localizer["NoLogs"]</em></p>
}
</TabPanel>
<TabPanel Name="Settings" Heading="Settings" ResourceKey="Settings">
<div class="container">
<div class="row mb-1 align-items-center">
<Label Class="col-sm-3" For="retention" HelpText="Number of days of events to retain" ResourceKey="Retention">Retention (Days): </Label>
<div class="col-sm-9">
<input id="retention" class="form-control" @bind="@_retention" />
</div>
</div>
</div>
<br />
<button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
</TabPanel>
</TabStrip>
}
@code {
@ -78,6 +95,7 @@ else
private string _function = "-";
private string _rows = "10";
private List<Log> _logs;
private string _retention = "";
public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Host;
@ -86,6 +104,7 @@ else
try
{
await GetLogs();
_retention = SettingService.GetSetting(PageState.Site.Settings, "LogRetention", "30");
}
catch (Exception ex)
{
@ -171,4 +190,22 @@ else
}
return classname;
}
private async Task SaveSiteSettings()
{
try
{
var settings = PageState.Site.Settings;
settings = SettingService.SetSetting(settings, "LogRetention", _retention);
await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
}
catch (Exception ex)
{
await logger.LogError(ex, "Error Saving Site Settings {Error}", ex.Message);
AddModuleMessage(Localizer["Error.SaveSiteSettings"], MessageType.Error);
}
}
}

33
Oqtane.Client/Modules/Admin/Visitors/Index.razor
View File

@ -2,6 +2,7 @@
@inherits ModuleBase
@inject IVisitorService VisitorService
@inject ISiteService SiteService
@inject ISettingService SettingService
@inject IStringLocalizer<Index> Localizer
@inject IStringLocalizer<SharedResources> SharedLocalizer
@ -60,14 +61,26 @@ else
<TabPanel Name="Settings" Heading="Settings" ResourceKey="Settings">
<div class="container">
<div class="row mb-1 align-items-center">
<Label Class="col-sm-3" For="visitortracking" HelpText="Specify if visitor tracking is enabled" ResourceKey="VisitorTracking">Visitor Tracking Enabled? </Label>
<Label Class="col-sm-3" For="tracking" HelpText="Specify if visitor tracking is enabled" ResourceKey="Tracking">Tracking Enabled? </Label>
<div class="col-sm-9">
<select id="visitortracking" class="form-select" @bind="@_visitortracking" >
<select id="tracking" class="form-select" @bind="@_tracking" >
<option value="True">@SharedLocalizer["Yes"]</option>
<option value="False">@SharedLocalizer["No"]</option>
</select>
</div>
</div>
<div class="row mb-1 align-items-center">
<Label Class="col-sm-3" For="filter" HelpText="Comma delimited list of terms which may exist in IP addresses, user agents, or languages which identify visitors which should not be tracked (ie. bots)" ResourceKey="Filter">Filter: </Label>
<div class="col-sm-9">
<textarea id="filter" class="form-control" @bind="@_filter" rows="3"></textarea>
</div>
</div>
<div class="row mb-1 align-items-center">
<Label Class="col-sm-3" For="retention" HelpText="Number of days of visitor activity to retain" ResourceKey="Retention">Retention (Days): </Label>
<div class="col-sm-9">
<input id="retention" class="form-control" @bind="@_retention" />
</div>
</div>
</div>
<br />
<button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
@ -79,14 +92,18 @@ else
private bool _users = false;
private int _days = 1;
private List<Visitor> _visitors;
private string _visitortracking;
private string _tracking;
private string _filter = "";
private string _retention = "";
public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Admin;
protected override async Task OnParametersSetAsync()
{
await GetVisitors();
_visitortracking = PageState.Site.VisitorTracking.ToString();
_tracking = PageState.Site.VisitorTracking.ToString();
_filter = SettingService.GetSetting(PageState.Site.Settings, "VisitorFilter", "");
_retention = SettingService.GetSetting(PageState.Site.Settings, "VisitorRetention", "30");
}
private async void TypeChanged(ChangeEventArgs e)
@ -131,8 +148,14 @@ else
try
{
var site = PageState.Site;
site.VisitorTracking = bool.Parse(_visitortracking);
site.VisitorTracking = bool.Parse(_tracking);
await SiteService.UpdateSiteAsync(site);
var settings = PageState.Site.Settings;
settings = SettingService.SetSetting(settings, "VisitorFilter", _filter);
settings = SettingService.SetSetting(settings, "VisitorRetention", _retention);
await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
}
catch (Exception ex)

18
Oqtane.Client/Resources/Modules/Admin/Logs/Index.resx
View File

@ -192,4 +192,22 @@
<data name="LogDetails.Text" xml:space="preserve">
<value>Details</value>
</data>
<data name="Error.SaveSiteSettings" xml:space="preserve">
<value>Error Saving Settings</value>
</data>
<data name="Events.Heading" xml:space="preserve">
<value>Events</value>
</data>
<data name="Retention.HelpText" xml:space="preserve">
<value>Number of days of events to retain</value>
</data>
<data name="Retention.Text" xml:space="preserve">
<value>Retention (Days):</value>
</data>
<data name="Settings.Heading" xml:space="preserve">
<value>Settings</value>
</data>
<data name="Success.SaveSiteSettings" xml:space="preserve">
<value>Settings Saved Successfully</value>
</data>
</root>

18
Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
View File

@ -156,11 +156,11 @@
<data name="Visitors.Heading" xml:space="preserve">
<value>Visitors</value>
</data>
<data name="VisitorTracking.HelpText" xml:space="preserve">
<data name="Tracking.HelpText" xml:space="preserve">
<value>Specify if visitor tracking is enabled</value>
</data>
<data name="VisitorTracking.Text" xml:space="preserve">
<value>Visitor Tracking Enabled?</value>
<data name="Tracking.Text" xml:space="preserve">
<value>Tracking Enabled?</value>
</data>
<data name="IP" xml:space="preserve">
<value>IP</value>
@ -174,4 +174,16 @@
<data name="Details.Text" xml:space="preserve">
<value>Details</value>
</data>
<data name="Filter.HelpText" xml:space="preserve">
<value>Comma delimited list of terms which may exist in IP addresses, user agents, or languages which identify visitors which should not be tracked (ie. bots)</value>
</data>
<data name="Filter.Text" xml:space="preserve">
<value>Filter:</value>
</data>
<data name="Retention.HelpText" xml:space="preserve">
<value>Number of days of visitor activity to retain</value>
</data>
<data name="Retention.Text" xml:space="preserve">
<value>Retention (Days):</value>
</data>
</root>

1
Oqtane.Client/Themes/Controls/Theme/ControlPanel.razor
View File

@ -399,6 +399,7 @@
await PageModuleService.UpdatePageModuleOrderAsync(pageModule.PageId, pageModule.Pane);
Message = $"<div class=\"alert alert-success mt-2 text-center\" role=\"alert\">{Localizer["Success.Page.ModuleAdd"]}</div>";
Title = "";
NavigationManager.NavigateTo(NavigateUrl());
}
else

76
Oqtane.Server/Infrastructure/Jobs/PurgeJob.cs Normal file
View File

@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Shared;
namespace Oqtane.Infrastructure
{
public class PurgeJob : HostedServiceBase
{
// JobType = "Oqtane.Infrastructure.PurgeJob, Oqtane.Server"
public PurgeJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory)
{
Name = "Purge Job";
Frequency = "d"; // daily
Interval = 1;
StartDate = DateTime.ParseExact("03:00", "H:mm", null, System.Globalization.DateTimeStyles.None); // 3 AM
IsEnabled = true;
}
// job is executed for each tenant in installation
public override string ExecuteJob(IServiceProvider provider)
{
string log = "";
// get services
var siteRepository = provider.GetRequiredService<ISiteRepository>();
var settingRepository = provider.GetRequiredService<ISettingRepository>();
var logRepository = provider.GetRequiredService<ILogRepository>();
var visitorRepository = provider.GetRequiredService<IVisitorRepository>();
// iterate through sites for current tenant
List<Site> sites = siteRepository.GetSites().ToList();
foreach (Site site in sites)
{
log += "Processing Site: " + site.Name + "<br />";
// get site settings
Dictionary<string, string> settings = GetSettings(settingRepository.GetSettings(EntityNames.Site, site.SiteId).ToList());
// purge event log
int logretention = 30;
if (settings.ContainsKey("LogRetention") && settings["LogRetention"] != "")
{
logretention = int.Parse(settings["LogRetention"]);
}
int count = logRepository.DeleteLogs(logretention);
log += count.ToString() + " Event Logs Purged<br />";
// purge visitors
int visitorrention = 30;
if (settings.ContainsKey("VisitorRetention") && settings["VisitorRetention"] != "")
{
visitorrention = int.Parse(settings["VisitorRetention"]);
}
count = visitorRepository.DeleteVisitors(visitorrention);
log += count.ToString() + " Visitors Purged<br />";
}
return log;
}
private Dictionary<string, string> GetSettings(List<Setting> settings)
{
Dictionary<string, string> dictionary = new Dictionary<string, string>();
foreach (Setting setting in settings.OrderBy(item => item.SettingName).ToList())
{
dictionary.Add(setting.SettingName, setting.SettingValue);
}
return dictionary;
}
}
}

18
Oqtane.Server/Pages/_Host.cshtml.cs
View File

@ -35,8 +35,9 @@ namespace Oqtane.Pages
private readonly IUrlMappingRepository _urlMappings;
private readonly IVisitorRepository _visitors;
private readonly IAliasRepository _aliases;
private readonly ISettingRepository _settings;
public HostModel(IConfiguration configuration, ITenantManager tenantManager, ILocalizationManager localizationManager, ILanguageRepository languages, IAntiforgery antiforgery, ISiteRepository sites, IPageRepository pages, IUrlMappingRepository urlMappings, IVisitorRepository visitors, IAliasRepository aliases)
public HostModel(IConfiguration configuration, ITenantManager tenantManager, ILocalizationManager localizationManager, ILanguageRepository languages, IAntiforgery antiforgery, ISiteRepository sites, IPageRepository pages, IUrlMappingRepository urlMappings, IVisitorRepository visitors, IAliasRepository aliases, ISettingRepository settings)
{
_configuration = configuration;
_tenantManager = tenantManager;
@ -48,6 +49,7 @@ namespace Oqtane.Pages
_urlMappings = urlMappings;
_visitors = visitors;
_aliases = aliases;
_settings = settings;
}
public string AntiForgeryToken = "";
@ -198,6 +200,20 @@ namespace Oqtane.Pages
language = (language.Contains(",")) ? language.Substring(0, language.IndexOf(",")) : language;
language = (language.Contains(";")) ? language.Substring(0, language.IndexOf(";")) : language;
language = (language.Trim().Length == 0) ? "*" : language;
// filter
var filter = _settings.GetSetting(EntityNames.Site, SiteId, "VisitorFilter");
if (filter != null && !string.IsNullOrEmpty(filter.SettingValue))
{
foreach (string term in filter.SettingValue.ToLower().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(sValue => sValue.Trim()).ToArray())
{
if (ip.ToLower().Contains(term) || useragent.ToLower().Contains(term) || language.ToLower().Contains(term))
{
return;
}
}
}
string url = Request.GetEncodedUrl();
string referrer = (Request.Headers[HeaderNames.Referer] != StringValues.Empty) ? Request.Headers[HeaderNames.Referer] : "";
int? userid = null;

3
Oqtane.Server/Repository/Interfaces/ILogRepository.cs
View File

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using Oqtane.Models;
namespace Oqtane.Repository
@ -8,5 +8,6 @@ namespace Oqtane.Repository
IEnumerable<Log> GetLogs(int siteId, string level, string function, int rows);
Log GetLog(int logId);
void AddLog(Log log);
int DeleteLogs(int age);
}
}

1
Oqtane.Server/Repository/Interfaces/ISettingRepository.cs
View File

@ -10,6 +10,7 @@ namespace Oqtane.Repository
Setting AddSetting(Setting setting);
Setting UpdateSetting(Setting setting);
Setting GetSetting(string entityName, int settingId);
Setting GetSetting(string entityName, int entityId, string settingName);
void DeleteSetting(string entityName, int settingId);
void DeleteSettings(string entityName, int entityId);
}

1
Oqtane.Server/Repository/Interfaces/IVisitorRepository.cs
View File

@ -11,5 +11,6 @@ namespace Oqtane.Repository
Visitor UpdateVisitor(Visitor visitor);
Visitor GetVisitor(int visitorId);
void DeleteVisitor(int visitorId);
int DeleteVisitors(int age);
}
}

21
Oqtane.Server/Repository/LogRepository.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using Oqtane.Models;
@ -47,5 +48,23 @@ namespace Oqtane.Repository
_db.Log.Add(log);
_db.SaveChanges();
}
public int DeleteLogs(int age)
{
// delete logs in batches of 100 records
int count = 0;
var purgedate = DateTime.Now.AddDays(-age);
var logs = _db.Log.Where(item => item.Level != "Error" && item.LogDate < purgedate)
.OrderBy(item => item.LogDate).Take(100).ToList();
while (logs.Count > 0)
{
count += logs.Count;
_db.Log.RemoveRange(logs);
_db.SaveChanges();
logs = _db.Log.Where(item => item.Level != "Error" && item.LogDate < purgedate)
.OrderBy(item => item.LogDate).Take(100).ToList();
}
return count;
}
}
}

12
Oqtane.Server/Repository/SettingRepository.cs
View File

@ -77,6 +77,18 @@ namespace Oqtane.Repository
}
}
public Setting GetSetting(string entityName, int entityId, string settingName)
{
if (IsMaster(entityName))
{
return _master.Setting.Where(item => item.EntityName == entityName && item.EntityId == entityId && item.SettingName == settingName).FirstOrDefault();
}
else
{
return _tenant.Setting.Where(item => item.EntityName == entityName && item.EntityId == entityId && item.SettingName == settingName).FirstOrDefault();
}
}
public void DeleteSetting(string entityName, int settingId)
{
if (IsMaster(entityName))

18
Oqtane.Server/Repository/VisitorRepository.cs
View File

@ -47,5 +47,23 @@ namespace Oqtane.Repository
_db.Visitor.Remove(visitor);
_db.SaveChanges();
}
public int DeleteVisitors(int age)
{
// delete visitors in batches of 100 records
int count = 0;
var purgedate = DateTime.Now.AddDays(-age);
var visitors = _db.Visitor.Where(item => item.Visits <= 1 && item.VisitedOn < purgedate)
.OrderBy(item => item.VisitedOn).Take(100).ToList();
while (visitors.Count > 0)
{
count += visitors.Count;
_db.Visitor.RemoveRange(visitors);
_db.SaveChanges();
visitors = _db.Visitor.Where(item => item.Visits < 2 && item.VisitedOn < purgedate)
.OrderBy(item => item.VisitedOn).Take(100).ToList();
}
return count;
}
}
}