Merge pull request #1912 from sbwalker/dev

include purge job for maintaining event logs and visitor logs

Oqtane.Client/Modules/Admin/Logs/Index.razor

@@ -1,6 +1,7 @@
 @namespace Oqtane.Modules.Admin.Logs
 @inherits ModuleBase
 @inject ILogService LogService
+@inject ISettingService SettingService
 @inject IStringLocalizer<Index> Localizer
 @inject IStringLocalizer<SharedResources> SharedLocalizer
 
@@ -10,6 +11,8 @@
 }
 else
 {
+    <TabStrip>
+        <TabPanel Name="Events" Heading="Events" ResourceKey="Events">
 			<div class="container g-0">
 				<div class="row mb-1 align-items-center">
 					<div class="col-sm-4">
@@ -71,6 +74,20 @@ else
 			{
 				<p><em>@Localizer["NoLogs"]</em></p>
 			}
+		</TabPanel>
+        <TabPanel Name="Settings" Heading="Settings" ResourceKey="Settings">
+			<div class="container">
+				<div class="row mb-1 align-items-center">
+					<Label Class="col-sm-3" For="retention" HelpText="Number of days of events to retain" ResourceKey="Retention">Retention (Days): </Label>
+					<div class="col-sm-9">
+						<input id="retention" class="form-control" @bind="@_retention" />
+					</div>
+				</div>
+			</div>
+			<br />
+			<button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
+		</TabPanel>
+	</TabStrip>
 }
 
 @code {
@@ -78,6 +95,7 @@ else
     private string _function = "-";
     private string _rows = "10";
     private List<Log> _logs;
+    private string _retention = "";
 
     public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Host;
 
@@ -86,6 +104,7 @@ else
         try
         {
             await GetLogs();
+			_retention = SettingService.GetSetting(PageState.Site.Settings, "LogRetention", "30");
         }
         catch (Exception ex)
         {
@@ -171,4 +190,22 @@ else
         }
         return classname;
     }
+
+	private async Task SaveSiteSettings()
+	{
+		try
+		{
+			var settings = PageState.Site.Settings;
+			settings = SettingService.SetSetting(settings, "LogRetention", _retention);
+            await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
+
+			AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
+		}
+		catch (Exception ex)
+		{
+			await logger.LogError(ex, "Error Saving Site Settings {Error}", ex.Message);
+			AddModuleMessage(Localizer["Error.SaveSiteSettings"], MessageType.Error);
+		}
+	}
+
 }

Oqtane.Client/Modules/Admin/Visitors/Index.razor

@@ -2,6 +2,7 @@
 @inherits ModuleBase
 @inject IVisitorService VisitorService
 @inject ISiteService SiteService
+@inject ISettingService SettingService
 @inject IStringLocalizer<Index> Localizer
 @inject IStringLocalizer<SharedResources> SharedLocalizer
 
@@ -60,14 +61,26 @@ else
         <TabPanel Name="Settings" Heading="Settings" ResourceKey="Settings">
 			<div class="container">
 				<div class="row mb-1 align-items-center">
-					<Label Class="col-sm-3" For="visitortracking" HelpText="Specify if visitor tracking is enabled" ResourceKey="VisitorTracking">Visitor Tracking Enabled? </Label>
+					<Label Class="col-sm-3" For="tracking" HelpText="Specify if visitor tracking is enabled" ResourceKey="Tracking">Tracking Enabled? </Label>
 					<div class="col-sm-9">
-						<select id="visitortracking" class="form-select" @bind="@_visitortracking" >
+						<select id="tracking" class="form-select" @bind="@_tracking" >
 							<option value="True">@SharedLocalizer["Yes"]</option>
 							<option value="False">@SharedLocalizer["No"]</option>
 						</select>
 					</div>
 				</div>
+				<div class="row mb-1 align-items-center">
+					<Label Class="col-sm-3" For="filter" HelpText="Comma delimited list of terms which may exist in IP addresses, user agents, or languages which identify visitors which should not be tracked (ie. bots)" ResourceKey="Filter">Filter: </Label>
+					<div class="col-sm-9">
+						<textarea id="filter" class="form-control" @bind="@_filter" rows="3"></textarea>
+					</div>
+				</div>
+				<div class="row mb-1 align-items-center">
+					<Label Class="col-sm-3" For="retention" HelpText="Number of days of visitor activity to retain" ResourceKey="Retention">Retention (Days): </Label>
+					<div class="col-sm-9">
+						<input id="retention" class="form-control" @bind="@_retention" />
+					</div>
+				</div>
 			</div>
 			<br />
 			<button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
@@ -79,14 +92,18 @@ else
 	private bool _users = false;
 	private int _days = 1;
 	private List<Visitor> _visitors;
-	private string _visitortracking;
+	private string _tracking;
+	private string _filter = "";
+	private string _retention = "";
 
 	public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Admin;
 
 	protected override async Task OnParametersSetAsync()
 	{
 		await GetVisitors();
-		_visitortracking = PageState.Site.VisitorTracking.ToString();
+		_tracking = PageState.Site.VisitorTracking.ToString();
+		_filter = SettingService.GetSetting(PageState.Site.Settings, "VisitorFilter", "");
+		_retention = SettingService.GetSetting(PageState.Site.Settings, "VisitorRetention", "30");
 	}
 
 	private async void TypeChanged(ChangeEventArgs e)
@@ -131,8 +148,14 @@ else
 		try
 		{
 			var site = PageState.Site;
-			site.VisitorTracking = bool.Parse(_visitortracking);
+			site.VisitorTracking = bool.Parse(_tracking);
 			await SiteService.UpdateSiteAsync(site);
+
+			var settings = PageState.Site.Settings;
+			settings = SettingService.SetSetting(settings, "VisitorFilter", _filter);
+			settings = SettingService.SetSetting(settings, "VisitorRetention", _retention);
+            await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
+
 			AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
 		}
 		catch (Exception ex)

Oqtane.Client/Resources/Modules/Admin/Logs/Index.resx

@@ -192,4 +192,22 @@
   <data name="LogDetails.Text" xml:space="preserve">
     <value>Details</value>
   </data>
+  <data name="Error.SaveSiteSettings" xml:space="preserve">
+    <value>Error Saving Settings</value>
+  </data>
+  <data name="Events.Heading" xml:space="preserve">
+    <value>Events</value>
+  </data>
+  <data name="Retention.HelpText" xml:space="preserve">
+    <value>Number of days of events to retain</value>
+  </data>
+  <data name="Retention.Text" xml:space="preserve">
+    <value>Retention (Days):</value>
+  </data>
+  <data name="Settings.Heading" xml:space="preserve">
+    <value>Settings</value>
+  </data>
+  <data name="Success.SaveSiteSettings" xml:space="preserve">
+    <value>Settings Saved Successfully</value>
+  </data>
 </root>

Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx

@@ -156,11 +156,11 @@
   <data name="Visitors.Heading" xml:space="preserve">
     <value>Visitors</value>
   </data>
-  <data name="VisitorTracking.HelpText" xml:space="preserve">
+  <data name="Tracking.HelpText" xml:space="preserve">
     <value>Specify if visitor tracking is enabled</value>
   </data>
-  <data name="VisitorTracking.Text" xml:space="preserve">
+  <data name="Tracking.Text" xml:space="preserve">
-    <value>Visitor Tracking Enabled?</value>
+    <value>Tracking Enabled?</value>
   </data>
   <data name="IP" xml:space="preserve">
     <value>IP</value>
@@ -174,4 +174,16 @@
   <data name="Details.Text" xml:space="preserve">
     <value>Details</value>
   </data>
+  <data name="Filter.HelpText" xml:space="preserve">
+    <value>Comma delimited list of terms which may exist in IP addresses, user agents, or languages which identify visitors which should not be tracked (ie. bots)</value>
+  </data>
+  <data name="Filter.Text" xml:space="preserve">
+    <value>Filter:</value>
+  </data>
+  <data name="Retention.HelpText" xml:space="preserve">
+    <value>Number of days of visitor activity to retain</value>
+  </data>
+  <data name="Retention.Text" xml:space="preserve">
+    <value>Retention (Days):</value>
+  </data>
 </root>

Oqtane.Client/Themes/Controls/Theme/ControlPanel.razor

@@ -399,6 +399,7 @@
 				await PageModuleService.UpdatePageModuleOrderAsync(pageModule.PageId, pageModule.Pane);
 
 				Message = $"<div class=\"alert alert-success mt-2 text-center\" role=\"alert\">{Localizer["Success.Page.ModuleAdd"]}</div>";
+				Title = "";
 				NavigationManager.NavigateTo(NavigateUrl());
             }
             else

Oqtane.Server/Infrastructure/Jobs/PurgeJob.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
+using Oqtane.Models;
+using Oqtane.Repository;
+using Oqtane.Shared;
+
+namespace Oqtane.Infrastructure
+{
+    public class PurgeJob : HostedServiceBase
+    {
+        // JobType = "Oqtane.Infrastructure.PurgeJob, Oqtane.Server"
+
+        public PurgeJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory)
+        {
+            Name = "Purge Job";
+            Frequency = "d"; // daily
+            Interval = 1;
+            StartDate = DateTime.ParseExact("03:00", "H:mm", null, System.Globalization.DateTimeStyles.None); // 3 AM
+            IsEnabled = true;
+        }
+
+        // job is executed for each tenant in installation
+        public override string ExecuteJob(IServiceProvider provider)
+        {
+            string log = "";
+
+            // get services
+            var siteRepository = provider.GetRequiredService<ISiteRepository>();
+            var settingRepository = provider.GetRequiredService<ISettingRepository>();
+            var logRepository = provider.GetRequiredService<ILogRepository>();
+            var visitorRepository = provider.GetRequiredService<IVisitorRepository>();
+
+            // iterate through sites for current tenant
+            List<Site> sites = siteRepository.GetSites().ToList();
+            foreach (Site site in sites)
+            {
+                log += "Processing Site: " + site.Name + "<br />";
+
+                // get site settings
+                Dictionary<string, string> settings = GetSettings(settingRepository.GetSettings(EntityNames.Site, site.SiteId).ToList());
+
+                // purge event log
+                int logretention = 30;
+                if (settings.ContainsKey("LogRetention") && settings["LogRetention"] != "")
+                {
+                    logretention = int.Parse(settings["LogRetention"]);
+                }
+                int count = logRepository.DeleteLogs(logretention);
+                log += count.ToString() + " Event Logs Purged<br />";
+
+                // purge visitors
+                int visitorrention = 30;
+                if (settings.ContainsKey("VisitorRetention") && settings["VisitorRetention"] != "")
+                {
+                    visitorrention = int.Parse(settings["VisitorRetention"]);
+                }
+                count = visitorRepository.DeleteVisitors(visitorrention);
+                log += count.ToString() + " Visitors Purged<br />";
+            }
+
+            return log;
+        }
+
+        private Dictionary<string, string> GetSettings(List<Setting> settings)
+        {
+            Dictionary<string, string> dictionary = new Dictionary<string, string>();
+            foreach (Setting setting in settings.OrderBy(item => item.SettingName).ToList())
+            {
+                dictionary.Add(setting.SettingName, setting.SettingValue);
+            }
+            return dictionary;
+        }
+    }
+}

Oqtane.Server/Pages/_Host.cshtml.cs

@@ -35,8 +35,9 @@ namespace Oqtane.Pages
         private readonly IUrlMappingRepository _urlMappings;
         private readonly IVisitorRepository _visitors;
         private readonly IAliasRepository _aliases;
+        private readonly ISettingRepository _settings;
 
-        public HostModel(IConfiguration configuration, ITenantManager tenantManager, ILocalizationManager localizationManager, ILanguageRepository languages, IAntiforgery antiforgery, ISiteRepository sites, IPageRepository pages, IUrlMappingRepository urlMappings, IVisitorRepository visitors, IAliasRepository aliases)
+        public HostModel(IConfiguration configuration, ITenantManager tenantManager, ILocalizationManager localizationManager, ILanguageRepository languages, IAntiforgery antiforgery, ISiteRepository sites, IPageRepository pages, IUrlMappingRepository urlMappings, IVisitorRepository visitors, IAliasRepository aliases, ISettingRepository settings)
         {
             _configuration = configuration;
             _tenantManager = tenantManager;
@@ -48,6 +49,7 @@ namespace Oqtane.Pages
             _urlMappings = urlMappings;
             _visitors = visitors;
             _aliases = aliases;
+            _settings = settings;
         }
 
         public string AntiForgeryToken = "";
@@ -198,6 +200,20 @@ namespace Oqtane.Pages
             language = (language.Contains(",")) ? language.Substring(0, language.IndexOf(",")) : language;
             language = (language.Contains(";")) ? language.Substring(0, language.IndexOf(";")) : language;
             language = (language.Trim().Length == 0) ? "*" : language;
+
+            // filter
+            var filter = _settings.GetSetting(EntityNames.Site, SiteId, "VisitorFilter");
+            if (filter != null && !string.IsNullOrEmpty(filter.SettingValue))
+            {
+                foreach (string term in filter.SettingValue.ToLower().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(sValue => sValue.Trim()).ToArray())
+                {
+                    if (ip.ToLower().Contains(term) || useragent.ToLower().Contains(term) || language.ToLower().Contains(term))
+                    {
+                        return;
+                    }
+                }
+            }
+
             string url = Request.GetEncodedUrl();
             string referrer = (Request.Headers[HeaderNames.Referer] != StringValues.Empty) ? Request.Headers[HeaderNames.Referer] : "";
             int? userid = null;

Oqtane.Server/Repository/Interfaces/ILogRepository.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using Oqtane.Models;
 
 namespace Oqtane.Repository
@@ -8,5 +8,6 @@ namespace Oqtane.Repository
         IEnumerable<Log> GetLogs(int siteId, string level, string function, int rows);
         Log GetLog(int logId);
         void AddLog(Log log);
+        int DeleteLogs(int age);
     }
 }

Oqtane.Server/Repository/Interfaces/ISettingRepository.cs

@@ -10,6 +10,7 @@ namespace Oqtane.Repository
         Setting AddSetting(Setting setting);
         Setting UpdateSetting(Setting setting);
         Setting GetSetting(string entityName, int settingId);
+        Setting GetSetting(string entityName, int entityId, string settingName);
         void DeleteSetting(string entityName, int settingId);
         void DeleteSettings(string entityName, int entityId);
     }

Oqtane.Server/Repository/Interfaces/IVisitorRepository.cs

@@ -11,5 +11,6 @@ namespace Oqtane.Repository
         Visitor UpdateVisitor(Visitor visitor);
         Visitor GetVisitor(int visitorId);
         void DeleteVisitor(int visitorId);
+        int DeleteVisitors(int age);
     }
 }

Oqtane.Server/Repository/LogRepository.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using Oqtane.Models;
 
@@ -47,5 +48,23 @@ namespace Oqtane.Repository
             _db.Log.Add(log);
             _db.SaveChanges();
         }
+
+        public int DeleteLogs(int age)
+        {
+            // delete logs in batches of 100 records
+            int count = 0;
+            var purgedate = DateTime.Now.AddDays(-age);
+            var logs = _db.Log.Where(item => item.Level != "Error" && item.LogDate < purgedate)
+                .OrderBy(item => item.LogDate).Take(100).ToList();
+            while (logs.Count > 0)
+            {
+                count += logs.Count;
+                _db.Log.RemoveRange(logs);
+                _db.SaveChanges();
+                logs = _db.Log.Where(item => item.Level != "Error" && item.LogDate < purgedate)
+                    .OrderBy(item => item.LogDate).Take(100).ToList();
+            }
+            return count;
+        }
     }
 }

Oqtane.Server/Repository/SettingRepository.cs

@@ -77,6 +77,18 @@ namespace Oqtane.Repository
             }
         }
 
+        public Setting GetSetting(string entityName, int entityId, string settingName)
+        {
+            if (IsMaster(entityName))
+            {
+                return _master.Setting.Where(item => item.EntityName == entityName && item.EntityId == entityId && item.SettingName == settingName).FirstOrDefault();
+            }
+            else
+            {
+                return _tenant.Setting.Where(item => item.EntityName == entityName && item.EntityId == entityId && item.SettingName == settingName).FirstOrDefault();
+            }
+        }
+
         public void DeleteSetting(string entityName, int settingId)
         {
             if (IsMaster(entityName))

Oqtane.Server/Repository/VisitorRepository.cs

@@ -47,5 +47,23 @@ namespace Oqtane.Repository
             _db.Visitor.Remove(visitor);
             _db.SaveChanges();
         }
+
+        public int DeleteVisitors(int age)
+        {
+            // delete visitors in batches of 100 records
+            int count = 0;
+            var purgedate = DateTime.Now.AddDays(-age);
+            var visitors = _db.Visitor.Where(item => item.Visits <= 1 && item.VisitedOn < purgedate)
+                .OrderBy(item => item.VisitedOn).Take(100).ToList();
+            while (visitors.Count > 0)
+            {
+                count += visitors.Count;
+                _db.Visitor.RemoveRange(visitors);
+                _db.SaveChanges();
+                visitors = _db.Visitor.Where(item => item.Visits < 2 && item.VisitedOn < purgedate)
+                    .OrderBy(item => item.VisitedOn).Take(100).ToList();
+            }
+            return count;
+        }
     }
 }