From 5b46dd7293997509d6f665dae635e484fa7c9b0b Mon Sep 17 00:00:00 2001 From: sbwalker Date: Sat, 6 Jul 2024 07:58:04 -0400 Subject: [PATCH] search refactoring --- .../Themes/Controls/Theme/LoginBase.cs | 2 +- Oqtane.Client/UI/SiteRouter.razor | 2 +- .../Infrastructure/Jobs/SearchIndexJob.cs | 221 +++++++++++++++--- .../Search/ModuleSearchIndexManager.cs | 145 ------------ .../Search/ModuleSearchResultManager.cs | 2 +- .../Managers/Search/PageSearchIndexManager.cs | 89 ------- .../Managers/Search/SearchIndexManagerBase.cs | 36 --- .../HtmlText/Manager/HtmlTextManager.cs | 25 +- Oqtane.Server/Pages/Sitemap.cshtml.cs | 5 +- .../Providers/DatabaseSearchProvider.cs | 15 +- Oqtane.Server/Services/SearchService.cs | 66 +----- Oqtane.Server/Services/SiteService.cs | 4 +- .../Interfaces/ISearchIndexManager.cs | 17 -- Oqtane.Shared/Interfaces/ISearchService.cs | 4 +- Oqtane.Shared/Interfaces/ISearchable.cs | 3 +- Oqtane.Shared/Models/SearchContent.cs | 3 + Oqtane.Shared/Shared/Constants.cs | 4 +- Oqtane.Shared/Shared/Utilities.cs | 11 +- 18 files changed, 248 insertions(+), 406 deletions(-) delete mode 100644 Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs delete mode 100644 Oqtane.Server/Managers/Search/PageSearchIndexManager.cs delete mode 100644 Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs delete mode 100644 Oqtane.Shared/Interfaces/ISearchIndexManager.cs diff --git a/Oqtane.Client/Themes/Controls/Theme/LoginBase.cs b/Oqtane.Client/Themes/Controls/Theme/LoginBase.cs index 7d7d575c..6e9d4bc9 100644 --- a/Oqtane.Client/Themes/Controls/Theme/LoginBase.cs +++ b/Oqtane.Client/Themes/Controls/Theme/LoginBase.cs @@ -59,7 +59,7 @@ namespace Oqtane.Themes.Controls logouturl = Utilities.TenantUrl(PageState.Alias, "/pages/logout/"); // verify anonymous users can access current page - if (UserSecurity.IsAuthorized(null, PermissionNames.View, PageState.Page.PermissionList) && Utilities.IsPageModuleVisible(PageState.Page.EffectiveDate, PageState.Page.ExpiryDate)) + if (UserSecurity.IsAuthorized(null, PermissionNames.View, PageState.Page.PermissionList) && Utilities.IsEffectiveOrExpired(PageState.Page.EffectiveDate, PageState.Page.ExpiryDate)) { returnurl = PageState.Route.PathAndQuery; } diff --git a/Oqtane.Client/UI/SiteRouter.razor b/Oqtane.Client/UI/SiteRouter.razor index 222344ae..b9cd2735 100644 --- a/Oqtane.Client/UI/SiteRouter.razor +++ b/Oqtane.Client/UI/SiteRouter.razor @@ -254,7 +254,7 @@ } // check if user is authorized to view page - if (page != null && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList))) + if (page != null && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) && (Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList))) { // edit mode if (user != null) diff --git a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs index 8ad3e194..6c1b2df0 100644 --- a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs +++ b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.Linq; -using System.Text; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using Oqtane.Interfaces; using Oqtane.Models; using Oqtane.Repository; using Oqtane.Services; @@ -11,7 +13,7 @@ namespace Oqtane.Infrastructure { public class SearchIndexJob : HostedServiceBase { - private const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; + private const string SearchLastIndexedOnSetting = "Search_LastIndexedOn"; public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory) { @@ -21,68 +23,231 @@ namespace Oqtane.Infrastructure IsEnabled = true; } - public override string ExecuteJob(IServiceProvider provider) + public override async Task ExecuteJobAsync(IServiceProvider provider) { + string log = ""; + // get services var siteRepository = provider.GetRequiredService(); var settingRepository = provider.GetRequiredService(); - var logRepository = provider.GetRequiredService(); + var tenantManager = provider.GetRequiredService(); + var pageRepository = provider.GetRequiredService(); + var pageModuleRepository = provider.GetRequiredService(); var searchService = provider.GetRequiredService(); var sites = siteRepository.GetSites().ToList(); - var logs = new StringBuilder(); - foreach (var site in sites) { - var startTime = GetSearchStartTime(site.SiteId, settingRepository); - logs.AppendLine($"Search: Begin index site: {site.Name}
"); + log += $"Indexing Site: {site.Name}
"; + + // initialize var currentTime = DateTime.UtcNow; + var lastIndexedOn = GetSearchLastIndexedOn(settingRepository, site.SiteId); + var tenantId = tenantManager.GetTenant().TenantId; + tenantManager.SetAlias(tenantId, site.SiteId); + var pages = pageRepository.GetPages(site.SiteId); + var pageModules = pageModuleRepository.GetPageModules(site.SiteId); + var searchContents = new List(); - searchService.IndexContent(site.SiteId, startTime, logNote => + // index pages + foreach (var page in pages) { - logs.AppendLine(logNote); - }, handleError => - { - logs.AppendLine(handleError); - }); + if (Constants.InternalPagePaths.Contains(page.Path)) + { + continue; + } - UpdateSearchStartTime(site.SiteId, currentTime, settingRepository); + bool changed = false; + bool removed = false; - logs.AppendLine($"Search: End index site: {site.Name}
"); + if (page.ModifiedOn >= lastIndexedOn) + { + changed = true; + removed = page.IsDeleted || !Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate); + + var searchContent = new SearchContent + { + SiteId = page.SiteId, + EntityName = EntityNames.Page, + EntityId = page.PageId.ToString(), + Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, + Description = string.Empty, + Body = $"{page.Name} {page.Title}", + Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}", + Permissions = $"{EntityNames.Page}:{page.PageId}", + ContentModifiedBy = page.ModifiedBy, + ContentModifiedOn = page.ModifiedOn, + AdditionalContent = string.Empty, + CreatedOn = DateTime.UtcNow, + IsDeleted = removed, + TenantId = tenantId + }; + searchContents.Add(searchContent); + } + + // index modules + foreach (var pageModule in pageModules.Where(item => item.PageId == page.PageId)) + { + if (pageModule.ModifiedOn >= lastIndexedOn) + { + changed = true; + } + + var searchable = false; + if (pageModule.Module.ModuleDefinition != null && pageModule.Module.ModuleDefinition.ServerManagerType != "") + { + Type type = Type.GetType(pageModule.Module.ModuleDefinition.ServerManagerType); + if (type?.GetInterface(nameof(ISearchable)) != null) + { + try + { + searchable = true; + + // determine if reindexing is necessary + var lastindexedon = (changed) ? DateTime.MinValue : lastIndexedOn; + + // index module content + var serverManager = (ISearchable)ActivatorUtilities.CreateInstance(provider, type); + var searchcontents = await serverManager.GetSearchContentsAsync(pageModule, lastindexedon); + if (searchcontents != null) + { + foreach (var searchContent in searchcontents) + { + SaveModuleMetaData(searchContent, pageModule, tenantId, removed); + searchContents.Add(searchContent); + } + } + } + catch (Exception ex) + { + log += ex.Message + "
"; + } + } + } + + if (!searchable && changed) + { + // module does not implement ISearchable + var searchContent = new SearchContent + { + SiteId = page.SiteId, + EntityName = EntityNames.Module, + EntityId = pageModule.ModuleId.ToString(), + Title = pageModule.Title, + Description = string.Empty, + Body = $"{pageModule.Title}", + Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}", + Permissions = $"{EntityNames.Module}:{pageModule.ModuleId},{EntityNames.Page}:{pageModule.PageId}", + ContentModifiedBy = pageModule.ModifiedBy, + ContentModifiedOn = pageModule.ModifiedOn, + AdditionalContent = string.Empty, + CreatedOn = DateTime.UtcNow, + IsDeleted = (removed || pageModule.IsDeleted || !Utilities.IsEffectiveOrExpired(pageModule.EffectiveDate, pageModule.ExpiryDate)), + TenantId = tenantId + }; + searchContents.Add(searchContent); + } + } + } + + // save search content + await searchService.SaveSearchContentAsync(searchContents); + log += $"Index Date: {lastIndexedOn}
"; + log += $"Items Indexed: {searchContents.Count}
"; + + // update last indexed on + SaveSearchLastIndexedOn(settingRepository, site.SiteId, currentTime); } - return logs.ToString(); + return log; } - private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository) + + private void SaveModuleMetaData(SearchContent searchContent, PageModule pageModule, int tenantId, bool removed) { - var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); - if(setting == null) + searchContent.SiteId = pageModule.Module.SiteId; + searchContent.TenantId = tenantId; + + if (string.IsNullOrEmpty(searchContent.EntityName)) { - return null; + searchContent.EntityName = EntityNames.Module; + } + + if (string.IsNullOrEmpty(searchContent.EntityId)) + { + searchContent.EntityId = pageModule.ModuleId.ToString(); + } + + if (string.IsNullOrEmpty(searchContent.Permissions)) + { + searchContent.Permissions = $"{EntityNames.Module}:{pageModule.ModuleId},{EntityNames.Page}:{pageModule.PageId}"; + } + + if (string.IsNullOrEmpty(searchContent.ContentModifiedBy)) + { + searchContent.ContentModifiedBy = pageModule.ModifiedBy; + } + + if (searchContent.ContentModifiedOn == DateTime.MinValue) + { + searchContent.ContentModifiedOn = pageModule.ModifiedOn; + } + + if (string.IsNullOrEmpty(searchContent.AdditionalContent)) + { + searchContent.AdditionalContent = string.Empty; + } + + if (pageModule.Page != null) + { + if (string.IsNullOrEmpty(searchContent.Url)) + { + searchContent.Url = $"{(!string.IsNullOrEmpty(pageModule.Page.Path) && !pageModule.Page.Path.StartsWith("/") ? "/" : "")}{pageModule.Page.Path}"; + } + + if (string.IsNullOrEmpty(searchContent.Title)) + { + searchContent.Title = !string.IsNullOrEmpty(pageModule.Page.Title) ? pageModule.Page.Title : pageModule.Page.Name; + } + } + + if (removed || pageModule.IsDeleted || !Utilities.IsEffectiveOrExpired(pageModule.EffectiveDate, pageModule.ExpiryDate)) + { + searchContent.IsDeleted = true; } - return Convert.ToDateTime(setting.SettingValue); } - private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository) + private DateTime GetSearchLastIndexedOn(ISettingRepository settingRepository, int siteId) { - var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchLastIndexedOnSetting); + if (setting != null) + { + return Convert.ToDateTime(setting.SettingValue); + } + else + { + return DateTime.MinValue; + } + } + + private void SaveSearchLastIndexedOn(ISettingRepository settingRepository, int siteId, DateTime lastIndexedOn) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchLastIndexedOnSetting); if (setting == null) { setting = new Setting { EntityName = EntityNames.Site, EntityId = siteId, - SettingName = SearchIndexStartTimeSettingName, - SettingValue = Convert.ToString(startTime), + SettingName = SearchLastIndexedOnSetting, + SettingValue = Convert.ToString(lastIndexedOn), }; - settingRepository.AddSetting(setting); } else { - setting.SettingValue = Convert.ToString(startTime); + setting.SettingValue = Convert.ToString(lastIndexedOn); settingRepository.UpdateSetting(setting); } } diff --git a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs deleted file mode 100644 index f5f1806b..00000000 --- a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Oqtane.Interfaces; -using Oqtane.Models; -using Oqtane.Repository; -using Oqtane.Shared; - -namespace Oqtane.Managers.Search -{ - public class ModuleSearchIndexManager : SearchIndexManagerBase - { - public const int ModuleSearchIndexManagerPriority = 200; - - private readonly IServiceProvider _serviceProvider; - private readonly ILogger _logger; - private readonly IPageModuleRepository _pageModuleRepostory; - private readonly IPageRepository _pageRepository; - - public ModuleSearchIndexManager( - IServiceProvider serviceProvider, - IPageModuleRepository pageModuleRepostory, - ILogger logger, - IPageRepository pageRepository) - : base(serviceProvider) - { - _serviceProvider = serviceProvider; - _logger = logger; - _pageModuleRepostory = pageModuleRepostory; - _pageRepository = pageRepository; - } - - public override string Name => EntityNames.Module; - - public override int Priority => ModuleSearchIndexManagerPriority; - - public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) - { - var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId); - var searchContentList = new List(); - - foreach(var pageModule in pageModules) - { - if(pageModule.Page == null || SearchUtils.IsSystemPage(pageModule.Page)) - { - continue; - } - - if (pageModule.Module.ModuleDefinition != null && pageModule.Module.ModuleDefinition.ServerManagerType != "") - { - _logger.LogDebug($"Search: Begin index module {pageModule.ModuleId}."); - var type = Type.GetType(pageModule.Module.ModuleDefinition.ServerManagerType); - if (type?.GetInterface(nameof(ISearchable)) != null) - { - try - { - var moduleSearch = (ISearchable)ActivatorUtilities.CreateInstance(_serviceProvider, type); - var contentList = moduleSearch.GetSearchContents(pageModule, startTime.GetValueOrDefault(DateTime.MinValue)); - if(contentList != null) - { - foreach(var searchContent in contentList) - { - SaveModuleMetaData(searchContent, pageModule); - - searchContentList.Add(searchContent); - } - } - - } - catch(Exception ex) - { - _logger.LogError(ex, $"Search: Index module {pageModule.ModuleId} failed."); - handleError($"Search: Index module {pageModule.ModuleId} failed: {ex.Message}"); - } - } - _logger.LogDebug($"Search: End index module {pageModule.ModuleId}."); - } - } - - processSearchContent(searchContentList); - - return searchContentList.Count; - } - - private void SaveModuleMetaData(SearchContent searchContent, PageModule pageModule) - { - searchContent.SiteId = pageModule.Module.SiteId; - - if(string.IsNullOrEmpty(searchContent.EntityName)) - { - searchContent.EntityName = EntityNames.Module; - } - - if(string.IsNullOrEmpty(searchContent.EntityId)) - { - searchContent.EntityId = pageModule.ModuleId.ToString(); - } - - if (string.IsNullOrEmpty(searchContent.Permissions)) - { - searchContent.Permissions = $"{EntityNames.Module}:{pageModule.ModuleId},{EntityNames.Page}:{pageModule.PageId}"; - } - - if (searchContent.ContentModifiedOn == DateTime.MinValue) - { - searchContent.ContentModifiedOn = pageModule.ModifiedOn; - } - - if (string.IsNullOrEmpty(searchContent.AdditionalContent)) - { - searchContent.AdditionalContent = string.Empty; - } - - if (pageModule.Page != null) - { - if (string.IsNullOrEmpty(searchContent.Url)) - { - searchContent.Url = $"{(!string.IsNullOrEmpty(pageModule.Page.Path) && !pageModule.Page.Path.StartsWith("/") ? "/" : "")}{pageModule.Page.Path}"; - } - - if (string.IsNullOrEmpty(searchContent.Title)) - { - searchContent.Title = !string.IsNullOrEmpty(pageModule.Page.Title) ? pageModule.Page.Title : pageModule.Page.Name; - } - } - - if (searchContent.SearchContentProperties == null) - { - searchContent.SearchContentProperties = new List(); - } - - if(!searchContent.SearchContentProperties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) - { - searchContent.SearchContentProperties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); - } - - if (!searchContent.SearchContentProperties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) - { - searchContent.SearchContentProperties.Add(new SearchContentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); - } - } - } -} diff --git a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs index d108962c..989a5333 100644 --- a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs +++ b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs @@ -54,7 +54,7 @@ namespace Oqtane.Managers.Search var page = pageRepository.GetPage(pageId); return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) - && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList)); + && (Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList)); } } } diff --git a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs deleted file mode 100644 index 9a74263b..00000000 --- a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs +++ /dev/null @@ -1,89 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Logging; -using Oqtane.Models; -using Oqtane.Repository; -using Oqtane.Shared; - -namespace Oqtane.Managers.Search -{ - public class PageSearchIndexManager : SearchIndexManagerBase - { - private const int PageSearchIndexManagerPriority = 100; - - private readonly IServiceProvider _serviceProvider; - private readonly ILogger _logger; - private readonly IPageRepository _pageRepository; - - public PageSearchIndexManager( - IServiceProvider serviceProvider, - ILogger logger, - IPageRepository pageRepository) - : base(serviceProvider) - { - _serviceProvider = serviceProvider; - _logger = logger; - _pageRepository = pageRepository; - } - - public override string Name => EntityNames.Page; - - public override int Priority => PageSearchIndexManagerPriority; - - public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) - { - var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue); - var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue); - var searchContentList = new List(); - - foreach(var page in pages) - { - try - { - if(SearchUtils.IsSystemPage(page)) - { - continue; - } - - var searchContent = new SearchContent - { - SiteId = page.SiteId, - EntityName = EntityNames.Page, - EntityId = page.PageId.ToString(), - Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, - Description = string.Empty, - Body = $"{page.Name} {page.Title}", - Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}", - Permissions = $"{EntityNames.Page}:{page.PageId}", - ContentModifiedBy = page.ModifiedBy, - ContentModifiedOn = page.ModifiedOn, - AdditionalContent = string.Empty, - CreatedOn = DateTime.UtcNow - }; - - if (searchContent.SearchContentProperties == null) - { - searchContent.SearchContentProperties = new List(); - } - - if (!searchContent.SearchContentProperties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) - { - searchContent.SearchContentProperties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); - } - - searchContentList.Add(searchContent); - } - catch(Exception ex) - { - _logger.LogError(ex, $"Search: Index page {page.PageId} failed."); - handleError($"Search: Index page {page.PageId} failed: {ex.Message}"); - } - } - - processSearchContent(searchContentList); - - return searchContentList.Count; - } - } -} diff --git a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs deleted file mode 100644 index a3898967..00000000 --- a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using Microsoft.Extensions.DependencyInjection; -using Oqtane.Models; -using Oqtane.Repository; -using Oqtane.Services; -using Oqtane.Shared; - -namespace Oqtane.Managers.Search -{ - public abstract class SearchIndexManagerBase : ISearchIndexManager - { - private const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; - - private readonly IServiceProvider _serviceProvider; - - public SearchIndexManagerBase(IServiceProvider serviceProvider) - { - _serviceProvider = serviceProvider; - } - - public abstract int Priority { get; } - - public abstract string Name { get; } - - public abstract int IndexContent(int siteId, DateTime? startDate, Action> processSearchContent, Action handleError); - - public virtual bool IsIndexEnabled(int siteId) - { - var settingName = string.Format(SearchIndexManagerEnabledSettingFormat, Name); - var settingRepository = _serviceProvider.GetRequiredService(); - var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName); - return setting == null || setting.SettingValue == "true"; - } - } -} diff --git a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs index 305fa22d..550c7104 100644 --- a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs +++ b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs @@ -11,6 +11,7 @@ using System.Linq; using Oqtane.Interfaces; using System.Collections.Generic; using System; +using System.Threading.Tasks; // ReSharper disable ConvertToUsingDeclaration @@ -48,23 +49,27 @@ namespace Oqtane.Modules.HtmlText.Manager return content; } - public List GetSearchContents(PageModule pageModule, DateTime startDate) + public async Task> GetSearchContentsAsync(PageModule pageModule, DateTime lastIndexedOn) { + await Task.CompletedTask; + var searchContentList = new List(); var htmltexts = _htmlText.GetHtmlTexts(pageModule.ModuleId); - if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate)) + if (htmltexts != null && htmltexts.Any()) { var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First(); - - searchContentList.Add(new SearchContent + if (htmltext.CreatedOn >= lastIndexedOn) { - Title = pageModule.Module.Title, - Description = string.Empty, - Body = htmltext.Content, - ContentModifiedBy = htmltext.ModifiedBy, - ContentModifiedOn = htmltext.ModifiedOn - }); + searchContentList.Add(new SearchContent + { + Title = pageModule.Module.Title, + Description = string.Empty, + Body = htmltext.Content, + ContentModifiedBy = htmltext.ModifiedBy, + ContentModifiedOn = htmltext.ModifiedOn + }); + } } return searchContentList; diff --git a/Oqtane.Server/Pages/Sitemap.cshtml.cs b/Oqtane.Server/Pages/Sitemap.cshtml.cs index 4debad0c..2435f831 100644 --- a/Oqtane.Server/Pages/Sitemap.cshtml.cs +++ b/Oqtane.Server/Pages/Sitemap.cshtml.cs @@ -46,16 +46,13 @@ namespace Oqtane.Pages { var sitemap = new List(); - // internal pages which should not be indexed - string[] internalPaths = { "login", "register", "reset", "404" }; - // build site map var rooturl = _alias.Protocol + (string.IsNullOrEmpty(_alias.Path) ? _alias.Name : _alias.Name.Substring(0, _alias.Name.IndexOf("/"))); var moduleDefinitions = _moduleDefinitions.GetModuleDefinitions(_alias.SiteId).ToList(); var pageModules = _pageModules.GetPageModules(_alias.SiteId); foreach (var page in _pages.GetPages(_alias.SiteId)) { - if (_userPermissions.IsAuthorized(null, PermissionNames.View, page.PermissionList) && !internalPaths.Contains(page.Path)) + if (_userPermissions.IsAuthorized(null, PermissionNames.View, page.PermissionList) && !Constants.InternalPagePaths.Contains(page.Path)) { var pageurl = rooturl; if (string.IsNullOrEmpty(page.Url)) diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs index 3b8b037d..29f04661 100644 --- a/Oqtane.Server/Providers/DatabaseSearchProvider.cs +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -46,16 +46,19 @@ namespace Oqtane.Providers public void SaveSearchContent(SearchContent searchContent, bool autoCommit = false) { - //remove exist document + //remove existing search content _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId); - //clean the search content to remove html tags - CleanSearchContent(searchContent); + if (!searchContent.IsDeleted) + { + //clean the search content to remove html tags + CleanSearchContent(searchContent); - _searchContentRepository.AddSearchContent(searchContent); + _searchContentRepository.AddSearchContent(searchContent); - //save the index words - AnalyzeSearchContent(searchContent); + //save the index words + AnalyzeSearchContent(searchContent); + } } public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs index 3ac21321..ecb8a231 100644 --- a/Oqtane.Server/Services/SearchService.cs +++ b/Oqtane.Server/Services/SearchService.cs @@ -44,45 +44,6 @@ namespace Oqtane.Services _cache = cache; } - public void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError) - { - var searchEnabled = SearchEnabled(siteId); - if(!searchEnabled) - { - logNote($"Search: Search is disabled on site {siteId}.
"); - return; - } - - _logger.LogDebug($"Search: Start Index Content of {siteId}, Start Time: {startTime.GetValueOrDefault(DateTime.MinValue)}"); - - var searchProvider = GetSearchProvider(siteId); - - SetTenant(siteId); - - if (startTime == null) - { - searchProvider.ResetIndex(); - } - - var searchIndexManagers = GetSearchIndexManagers(m => { }); - foreach (var searchIndexManager in searchIndexManagers) - { - if (!searchIndexManager.IsIndexEnabled(siteId)) - { - logNote($"Search: Ignore indexer {searchIndexManager.Name} because it's disabled.
"); - } - else - { - _logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}"); - - var count = searchIndexManager.IndexContent(siteId, startTime, SaveSearchContent, handleError); - logNote($"Search: Indexer {searchIndexManager.Name} processed {count} search content.
"); - - _logger.LogDebug($"Search: End Index {searchIndexManager.Name}"); - } - } - } - public async Task SearchAsync(SearchQuery searchQuery) { var searchProvider = GetSearchProvider(searchQuery.SiteId); @@ -141,23 +102,6 @@ namespace Oqtane.Services _tenantManager.SetAlias(alias); } - private List GetSearchIndexManagers(Action initManager) - { - var managers = new List(); - var managerTypes = AppDomain.CurrentDomain.GetAssemblies() - .SelectMany(s => s.GetTypes()) - .Where(p => typeof(ISearchIndexManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); - - foreach (var type in managerTypes) - { - var manager = (ISearchIndexManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); - initManager(manager); - managers.Add(manager); - } - - return managers.OrderBy(i => i.Priority).ToList(); - } - private List GetSearchResultManagers() { var managers = new List(); @@ -174,13 +118,13 @@ namespace Oqtane.Services return managers.ToList(); } - private void SaveSearchContent(List searchContentList) + public async Task SaveSearchContentAsync(List searchContents) { - if(searchContentList.Any()) + if(searchContents.Any()) { - var searchProvider = GetSearchProvider(searchContentList.First().SiteId); + var searchProvider = GetSearchProvider(searchContents.First().SiteId); - foreach (var searchContent in searchContentList) + foreach (var searchContent in searchContents) { try { @@ -192,6 +136,8 @@ namespace Oqtane.Services } } + await Task.CompletedTask; + //commit the index changes searchProvider.Commit(); } diff --git a/Oqtane.Server/Services/SiteService.cs b/Oqtane.Server/Services/SiteService.cs index 97198373..ccb4c97e 100644 --- a/Oqtane.Server/Services/SiteService.cs +++ b/Oqtane.Server/Services/SiteService.cs @@ -72,7 +72,7 @@ namespace Oqtane.Services var pages = new List(); foreach (Page page in site.Pages) { - if (!page.IsDeleted && _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.View, page.PermissionList) && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.Edit, page.PermissionList))) + if (!page.IsDeleted && _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.View, page.PermissionList) && (Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate) || _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.Edit, page.PermissionList))) { pages.Add(page); } @@ -252,7 +252,7 @@ namespace Oqtane.Services var modules = new List(); foreach (Module module in sitemodules.Where(item => (item.PageId == pageId || pageId == -1) && !item.IsDeleted && _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.View, item.PermissionList))) { - if (Utilities.IsPageModuleVisible(module.EffectiveDate, module.ExpiryDate) || _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.Edit, module.PermissionList)) + if (Utilities.IsEffectiveOrExpired(module.EffectiveDate, module.ExpiryDate) || _userPermissions.IsAuthorized(_accessor.HttpContext.User, PermissionNames.Edit, module.PermissionList)) { modules.Add(module); } diff --git a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs deleted file mode 100644 index 576176fd..00000000 --- a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using Oqtane.Models; - -namespace Oqtane.Services -{ - public interface ISearchIndexManager - { - int Priority { get; } - - string Name { get; } - - bool IsIndexEnabled(int siteId); - - int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError); - } -} diff --git a/Oqtane.Shared/Interfaces/ISearchService.cs b/Oqtane.Shared/Interfaces/ISearchService.cs index 0f125c65..133fac0a 100644 --- a/Oqtane.Shared/Interfaces/ISearchService.cs +++ b/Oqtane.Shared/Interfaces/ISearchService.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections; using System.Collections.Generic; using System.Threading.Tasks; using Oqtane.Models; @@ -8,7 +6,7 @@ namespace Oqtane.Services { public interface ISearchService { - void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError); + Task SaveSearchContentAsync(List searchContents); Task SearchAsync(SearchQuery searchQuery); } diff --git a/Oqtane.Shared/Interfaces/ISearchable.cs b/Oqtane.Shared/Interfaces/ISearchable.cs index 33f81d51..dbd7c47e 100644 --- a/Oqtane.Shared/Interfaces/ISearchable.cs +++ b/Oqtane.Shared/Interfaces/ISearchable.cs @@ -1,11 +1,12 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Oqtane.Models; namespace Oqtane.Interfaces { public interface ISearchable { - public List GetSearchContents(PageModule pageModule, DateTime startTime); + public Task> GetSearchContentsAsync(PageModule pageModule, DateTime lastIndexedOn); } } diff --git a/Oqtane.Shared/Models/SearchContent.cs b/Oqtane.Shared/Models/SearchContent.cs index 77f803a9..03c97712 100644 --- a/Oqtane.Shared/Models/SearchContent.cs +++ b/Oqtane.Shared/Models/SearchContent.cs @@ -42,6 +42,9 @@ namespace Oqtane.Models [NotMapped] public int TenantId { get; set; } + [NotMapped] + public bool IsDeleted { get; set; } + // constructors public SearchContent() { } diff --git a/Oqtane.Shared/Shared/Constants.cs b/Oqtane.Shared/Shared/Constants.cs index 3abb1ed5..7b70e1ff 100644 --- a/Oqtane.Shared/Shared/Constants.cs +++ b/Oqtane.Shared/Shared/Constants.cs @@ -81,7 +81,9 @@ namespace Oqtane.Shared public const string DefaultSearchProviderName = "Database"; public const string SearchPageIdPropertyName = "PageId"; public const string SearchModuleIdPropertyName = "ModuleId"; - + + public static readonly string[] InternalPagePaths = { "login", "register", "reset", "404" }; + // Obsolete constants const string RoleObsoleteMessage = "Use the corresponding member from Oqtane.Shared.RoleNames"; diff --git a/Oqtane.Shared/Shared/Utilities.cs b/Oqtane.Shared/Shared/Utilities.cs index 5498a87a..3b8b012a 100644 --- a/Oqtane.Shared/Shared/Utilities.cs +++ b/Oqtane.Shared/Shared/Utilities.cs @@ -560,7 +560,7 @@ namespace Oqtane.Shared return (localDateTime?.Date, localTime); } - public static bool IsPageModuleVisible(DateTime? effectiveDate, DateTime? expiryDate) + public static bool IsEffectiveOrExpired(DateTime? effectiveDate, DateTime? expiryDate) { DateTime currentUtcTime = DateTime.UtcNow; @@ -582,6 +582,7 @@ namespace Oqtane.Shared return true; } } + public static bool ValidateEffectiveExpiryDates(DateTime? effectiveDate, DateTime? expiryDate) { // Treat DateTime.MinValue as null @@ -609,6 +610,7 @@ namespace Oqtane.Shared return true; } } + [Obsolete("ContentUrl(Alias alias, int fileId) is deprecated. Use FileUrl(Alias alias, int fileId) instead.", false)] public static string ContentUrl(Alias alias, int fileId) { @@ -623,5 +625,12 @@ namespace Oqtane.Shared return $"{alias?.BaseUrl}{aliasUrl}{Constants.ContentUrl}{fileId}{method}"; } + + [Obsolete("IsPageModuleVisible(DateTime?, DateTime?) is deprecated. Use IsEffectiveOrExpired(DateTime?, DateTime?) instead.", false)] + public static bool IsPageModuleVisible(DateTime? effectiveDate, DateTime? expiryDate) + { + return IsEffectiveOrExpired(effectiveDate, expiryDate); + } + } }