From 9d85ca07f47c9ee5abad20190868a720e66b771d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 3 Jun 2024 21:19:42 +0800 Subject: [PATCH 1/5] #4303: add search function. --- .../Modules/SearchResults/Index.razor | 147 ++++++++++++ .../Modules/SearchResults/ModuleInfo.cs | 25 ++ .../Services/ISearchResultsService.cs | 13 ++ .../Services/SearchResultsService.cs | 23 ++ .../Modules/SearchResults/Settings.razor | 46 ++++ .../Modules/SearchResults/Index.resx | 156 +++++++++++++ .../Modules/SearchResults/Settings.resx | 123 ++++++++++ .../Resources/Themes/Controls/Search.resx | 126 +++++++++++ .../Themes/BlazorTheme/Themes/Default.razor | 4 +- .../Themes/Controls/Theme/Search.razor | 57 +++++ .../Themes/OqtaneTheme/Themes/Default.razor | 7 +- .../OqtaneServiceCollectionExtensions.cs | 4 + .../Infrastructure/Jobs/SearchIndexJob.cs | 88 +++++++ .../SiteTemplates/DefaultSiteTemplate.cs | 24 ++ .../Search/ModuleSearchIndexManager.cs | 127 +++++++++++ .../Search/ModuleSearchResultManager.cs | 69 ++++++ .../Managers/Search/PageSearchIndexManager.cs | 94 ++++++++ .../Search/PageSearchResultManager.cs | 46 ++++ .../Managers/Search/SearchIndexManagerBase.cs | 34 +++ .../SearchDocumentEntityBuilder.cs | 64 ++++++ .../SearchDocumentPropertyEntityBuilder.cs | 40 ++++ .../SearchDocumentTagEntityBuilder.cs | 37 +++ .../Tenant/05020001_AddSearchTables.cs | 42 ++++ .../HtmlText/Manager/HtmlTextManager.cs | 34 ++- .../Controllers/SearchResultsController.cs | 43 ++++ .../Services/SearchResultsService.cs | 36 +++ .../SearchResults/Startup/ServerStartup.cs | 24 ++ .../Providers/DatabaseSearchProvider.cs | 202 +++++++++++++++++ .../Repository/Context/TenantDBContext.cs | 3 + .../Interfaces/ISearchDocumentRepository.cs | 17 ++ .../Repository/SearchDocumentRepository.cs | 136 +++++++++++ Oqtane.Server/Services/SearchService.cs | 214 ++++++++++++++++++ .../Oqtane.Modules.SearchResults/Module.css | 3 + .../Oqtane.Themes.OqtaneTheme/Theme.css | 4 + Oqtane.Server/wwwroot/css/app.css | 14 ++ Oqtane.Shared/Enums/SearchSortDirections.cs | 10 + Oqtane.Shared/Enums/SearchSortFields.cs | 11 + Oqtane.Shared/Interfaces/IModuleSearch.cs | 14 ++ .../Interfaces/ISearchIndexManager.cs | 20 ++ Oqtane.Shared/Interfaces/ISearchProvider.cs | 26 +++ .../Interfaces/ISearchResultManager.cs | 14 ++ Oqtane.Shared/Interfaces/ISearchService.cs | 15 ++ Oqtane.Shared/Models/SearchDocument.cs | 45 ++++ .../Models/SearchDocumentProperty.cs | 17 ++ Oqtane.Shared/Models/SearchDocumentTag.cs | 14 ++ Oqtane.Shared/Models/SearchQuery.cs | 37 +++ Oqtane.Shared/Models/SearchResult.cs | 11 + Oqtane.Shared/Models/SearchResults.cs | 11 + Oqtane.Shared/Shared/Constants.cs | 16 ++ Oqtane.Shared/Shared/SearchUtils.cs | 95 ++++++++ 50 files changed, 2478 insertions(+), 4 deletions(-) create mode 100644 Oqtane.Client/Modules/SearchResults/Index.razor create mode 100644 Oqtane.Client/Modules/SearchResults/ModuleInfo.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Settings.razor create mode 100644 Oqtane.Client/Resources/Modules/SearchResults/Index.resx create mode 100644 Oqtane.Client/Resources/Modules/SearchResults/Settings.resx create mode 100644 Oqtane.Client/Resources/Themes/Controls/Search.resx create mode 100644 Oqtane.Client/Themes/Controls/Theme/Search.razor create mode 100644 Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs create mode 100644 Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs create mode 100644 Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs create mode 100644 Oqtane.Server/Managers/Search/PageSearchIndexManager.cs create mode 100644 Oqtane.Server/Managers/Search/PageSearchResultManager.cs create mode 100644 Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs create mode 100644 Oqtane.Server/Providers/DatabaseSearchProvider.cs create mode 100644 Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs create mode 100644 Oqtane.Server/Repository/SearchDocumentRepository.cs create mode 100644 Oqtane.Server/Services/SearchService.cs create mode 100644 Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css create mode 100644 Oqtane.Shared/Enums/SearchSortDirections.cs create mode 100644 Oqtane.Shared/Enums/SearchSortFields.cs create mode 100644 Oqtane.Shared/Interfaces/IModuleSearch.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchIndexManager.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchProvider.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchResultManager.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchService.cs create mode 100644 Oqtane.Shared/Models/SearchDocument.cs create mode 100644 Oqtane.Shared/Models/SearchDocumentProperty.cs create mode 100644 Oqtane.Shared/Models/SearchDocumentTag.cs create mode 100644 Oqtane.Shared/Models/SearchQuery.cs create mode 100644 Oqtane.Shared/Models/SearchResult.cs create mode 100644 Oqtane.Shared/Models/SearchResults.cs create mode 100644 Oqtane.Shared/Shared/SearchUtils.cs diff --git a/Oqtane.Client/Modules/SearchResults/Index.razor b/Oqtane.Client/Modules/SearchResults/Index.razor new file mode 100644 index 00000000..8ae0c087 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Index.razor @@ -0,0 +1,147 @@ +@using Oqtane.Modules.SearchResults.Services +@namespace Oqtane.Modules.SearchResults +@inherits ModuleBase +@inject ISearchResultsService SearchResultsService +@inject IStringLocalizer Localizer + +
+
+
+
+ @Localizer["SearchPrefix"] + + +
+
+
+
+
+ @if (_loading) + { +
+ } + else + { + @if (_searchResults != null && _searchResults.Results != null) + { + if (_searchResults.Results.Any()) + { + + +
+

@context.Title

+
@context.Url
+

@((MarkupString)context.Snippet)

+
+ + + } + else + { + + } + } +
+ } +
+
+
+@code { + private SearchSortDirections _searchSortDirection = SearchSortDirections.Descending; //default sort by + private SearchSortFields _searchSortField = SearchSortFields.Relevance; + private string _keywords; + private bool _loading; + private SearchResults _searchResults; + private int _currentPage = 0; + private int _pageSize = Constants.SearchDefaultPageSize; + private int _displayPages = 7; + + protected override async Task OnInitializedAsync() + { + if (ModuleState.Settings.ContainsKey("PageSize")) + { + _pageSize = int.Parse(ModuleState.Settings["PageSize"]); + } + + if (PageState.QueryString.ContainsKey("s")) + { + _keywords = PageState.QueryString["s"]; + } + + if (PageState.QueryString.ContainsKey("p")) + { + _currentPage = Convert.ToInt32(PageState.QueryString["p"]); + if (_currentPage < 1) + { + _currentPage = 1; + } + } + + if (!string.IsNullOrEmpty(_keywords)) + { + await PerformSearch(); + } + } + + private async Task KeywordsChanged(KeyboardEventArgs e) + { + if (e.Code == "Enter" || e.Code == "NumpadEnter") + { + if (!string.IsNullOrEmpty(_keywords)) + { + await Search(); + } + } + } + + private async Task Search() + { + if (string.IsNullOrEmpty(_keywords)) + { + AddModuleMessage(Localizer["MissingKeywords"], MessageType.Warning); + } + else + { + ClearModuleMessage(); + + + _currentPage = 0; + await PerformSearch(); + } + } + + private async Task PerformSearch() + { + _loading = true; + StateHasChanged(); + + var searchQuery = new SearchQuery + { + SiteId = PageState.Site.SiteId, + Alias = PageState.Alias, + User = PageState.User, + Keywords = _keywords, + SortDirection = _searchSortDirection, + SortField = _searchSortField, + PageIndex = 0, + PageSize = int.MaxValue + }; + + _searchResults = await SearchResultsService.SearchAsync(PageState.ModuleId, searchQuery); + + _loading = false; + StateHasChanged(); + } +} \ No newline at end of file diff --git a/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs b/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs new file mode 100644 index 00000000..ae4bc7d2 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class ModuleInfo : IModule + { + public ModuleDefinition ModuleDefinition => new ModuleDefinition + { + Name = "Search Results", + Description = "Display Search Results", + Version = "1.0.0", + ServerManagerType = "", + ReleaseVersions = "1.0.0", + SettingsType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client", + Resources = new List() + { + new Resource { ResourceType = ResourceType.Stylesheet, Url = "~/Module.css" } + } + }; + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs b/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs new file mode 100644 index 00000000..2f1c1308 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public interface ISearchResultsService + { + Task SearchAsync(int moduleId, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs new file mode 100644 index 00000000..3b241ea4 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class SearchResultsService : ServiceBase, ISearchResultsService, IClientService + { + public SearchResultsService(HttpClient http, SiteState siteState) : base(http, siteState) {} + + private string ApiUrl => CreateApiUrl("SearchResults"); + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + return await PostJsonAsync(CreateAuthorizationPolicyUrl(ApiUrl, EntityNames.Module, moduleId), searchQuery); + } + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Settings.razor b/Oqtane.Client/Modules/SearchResults/Settings.razor new file mode 100644 index 00000000..b38d2946 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Settings.razor @@ -0,0 +1,46 @@ +@namespace Oqtane.Modules.SearchResults +@inherits ModuleBase +@inject ISettingService SettingService +@implements Oqtane.Interfaces.ISettingsControl +@inject IStringLocalizer Localizer +@inject IStringLocalizer SharedLocalizer + +
+
+ +
+ +
+
+
+ +@code { + private string resourceType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client"; // for localization + private string _pageSize; + + protected override void OnInitialized() + { + try + { + _pageSize = SettingService.GetSetting(ModuleState.Settings, "PageSize", Constants.SearchDefaultPageSize.ToString()); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } + + public async Task UpdateSettings() + { + try + { + var settings = await SettingService.GetModuleSettingsAsync(ModuleState.ModuleId); + settings = SettingService.SetSetting(settings, "PageSize", _pageSize); + await SettingService.UpdateModuleSettingsAsync(settings, ModuleState.ModuleId); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } +} diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Index.resx b/Oqtane.Client/Resources/Modules/SearchResults/Index.resx new file mode 100644 index 00000000..bd07f3f8 --- /dev/null +++ b/Oqtane.Client/Resources/Modules/SearchResults/Index.resx @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Ascending + + + Descending + + + Please provide the search keywords. + + + Modification Time + + + No results found. + + + Relevance + + + Search + + + Search + + + Search: + + + Sort Direction + + + Sort By + + + Title + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx b/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx new file mode 100644 index 00000000..1c48707d --- /dev/null +++ b/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Page Size + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Themes/Controls/Search.resx b/Oqtane.Client/Resources/Themes/Controls/Search.resx new file mode 100644 index 00000000..c2d2b4ee --- /dev/null +++ b/Oqtane.Client/Resources/Themes/Controls/Search.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Search + + + Search + + \ No newline at end of file diff --git a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor index 553d97d1..2e0da4d3 100644 --- a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor @@ -9,7 +9,9 @@
diff --git a/Oqtane.Client/Themes/Controls/Theme/Search.razor b/Oqtane.Client/Themes/Controls/Theme/Search.razor new file mode 100644 index 00000000..dbd28b40 --- /dev/null +++ b/Oqtane.Client/Themes/Controls/Theme/Search.razor @@ -0,0 +1,57 @@ +@namespace Oqtane.Themes.Controls +@using System.Net +@using Microsoft.AspNetCore.Http +@inherits ThemeControlBase +@inject IStringLocalizer Localizer +@inject NavigationManager NavigationManager +@inject IHttpContextAccessor HttpContext + +@if (_searchResultsPage != null) +{ + +
+ + + +
+ +} + + + +@code { + private const string SearchResultPagePath = "search-results"; + + private Page _searchResultsPage; + private string _keywords = ""; + + [Parameter] + public string CssClass { get; set; } + + protected override void OnInitialized() + { + _searchResultsPage = PageState.Pages.FirstOrDefault(i => i.Path == SearchResultPagePath); + } + + protected override void OnParametersSet() + { + } + + private void PerformSearch() + { + var keywords = HttpContext.HttpContext.Request.Form["keywords"]; + if (!string.IsNullOrEmpty(keywords) && _searchResultsPage != null) + { + var url = NavigateUrl(_searchResultsPage.Path, $"s={keywords}"); + NavigationManager.NavigateTo(url); + } + } +} + + diff --git a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor index 261597aa..936c823e 100644 --- a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor @@ -6,7 +6,12 @@
diff --git a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs index 19eb4cbe..a6612866 100644 --- a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs +++ b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs @@ -21,6 +21,7 @@ using Oqtane.Infrastructure; using Oqtane.Infrastructure.Interfaces; using Oqtane.Managers; using Oqtane.Modules; +using Oqtane.Providers; using Oqtane.Repository; using Oqtane.Security; using Oqtane.Services; @@ -97,6 +98,8 @@ namespace Microsoft.Extensions.DependencyInjection services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); return services; } @@ -131,6 +134,7 @@ namespace Microsoft.Extensions.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); // managers services.AddTransient(); diff --git a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs new file mode 100644 index 00000000..e6b7f216 --- /dev/null +++ b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs @@ -0,0 +1,88 @@ +using System; +using System.Linq; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Infrastructure +{ + public class SearchIndexJob : HostedServiceBase + { + public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory) + { + Name = "Search Index Job"; + Frequency = "m"; // run every minute. + Interval = 1; + IsEnabled = true; + } + + public override string ExecuteJob(IServiceProvider provider) + { + // get services + var siteRepository = provider.GetRequiredService(); + var settingRepository = provider.GetRequiredService(); + var logRepository = provider.GetRequiredService(); + var searchService = provider.GetRequiredService(); + + var sites = siteRepository.GetSites().ToList(); + var logs = new StringBuilder(); + + foreach (var site in sites) + { + var startTime = GetSearchStartTime(site.SiteId, settingRepository); + logs.AppendLine($"Search: Begin index site: {site.Name}
"); + var currentTime = DateTime.UtcNow; + + searchService.IndexContent(site.SiteId, startTime, logNote => + { + logs.AppendLine(logNote); + }, handleError => + { + logs.AppendLine(handleError); + }); + + UpdateSearchStartTime(site.SiteId, currentTime, settingRepository); + + logs.AppendLine($"Search: End index site: {site.Name}
"); + } + + return logs.ToString(); + } + + private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + if(setting == null) + { + return null; + } + + return Convert.ToDateTime(setting.SettingValue); + } + + private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + if (setting == null) + { + setting = new Setting + { + EntityName = EntityNames.Site, + EntityId = siteId, + SettingName = Constants.SearchIndexStartTimeSettingName, + SettingValue = Convert.ToString(startTime), + }; + + settingRepository.AddSetting(setting); + } + else + { + setting.SettingValue = Convert.ToString(startTime); + settingRepository.UpdateSetting(setting); + } + } + } +} diff --git a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs index 7cf27808..99bb8aaa 100644 --- a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs +++ b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs @@ -133,6 +133,30 @@ namespace Oqtane.SiteTemplates } } }); + _pageTemplates.Add(new PageTemplate + { + Name = "Search Results", + Parent = "", + Order = 7, + Path = "search-results", + Icon = "oi oi-magnifying-glass", + IsNavigation = false, + IsPersonalizable = false, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + }, + PageTemplateModules = new List { + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + } + } + } + }); if (System.IO.File.Exists(Path.Combine(_environment.WebRootPath, "images", "logo-white.png"))) { diff --git a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs new file mode 100644 index 00000000..fec739d2 --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchIndexManager : SearchIndexManagerBase + { + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageModuleRepository _pageModuleRepostory; + private readonly IPageRepository _pageRepository; + + public ModuleSearchIndexManager( + IServiceProvider serviceProvider, + IPageModuleRepository pageModuleRepostory, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageModuleRepostory = pageModuleRepostory; + _pageRepository = pageRepository; + } + + public override string Name => Constants.ModuleSearchIndexManagerName; + + public override int Priority => Constants.ModuleSearchIndexManagerPriority; + + public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + { + var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId); + var searchDocuments = new List(); + + foreach(var pageModule in pageModules) + { + var module = pageModule.Module; + if (module.ModuleDefinition.ServerManagerType != "") + { + _logger.LogDebug($"Search: Begin index module {module.ModuleId}."); + var type = Type.GetType(module.ModuleDefinition.ServerManagerType); + if (type?.GetInterface("IModuleSearch") != null) + { + try + { + var moduleSearch = (IModuleSearch)ActivatorUtilities.CreateInstance(_serviceProvider, type); + var documents = moduleSearch.GetSearchDocuments(module, startTime.GetValueOrDefault(DateTime.MinValue)); + if(documents != null) + { + foreach(var document in documents) + { + SaveModuleMetaData(document, pageModule); + + searchDocuments.Add(document); + } + } + + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index module {module.ModuleId} failed."); + handleError($"Search: Index module {module.ModuleId} failed: {ex.Message}"); + } + } + _logger.LogDebug($"Search: End index module {module.ModuleId}."); + } + } + + processSearchDocuments(searchDocuments); + + return searchDocuments.Count; + } + + private void SaveModuleMetaData(SearchDocument document, PageModule pageModule) + { + + document.EntryId = pageModule.ModuleId; + document.IndexerName = Name; + document.SiteId = pageModule.Module.SiteId; + document.LanguageCode = string.Empty; + + if(document.ModifiedTime == DateTime.MinValue) + { + document.ModifiedTime = pageModule.ModifiedOn; + } + + if (string.IsNullOrEmpty(document.AdditionalContent)) + { + document.AdditionalContent = string.Empty; + } + + var page = _pageRepository.GetPage(pageModule.PageId); + + if (string.IsNullOrEmpty(document.Url) && page != null) + { + document.Url = page.Url; + } + + if (string.IsNullOrEmpty(document.Title) && page != null) + { + document.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name; + } + + if (document.Properties == null) + { + document.Properties = new List(); + } + + if(!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); + } + + if (!document.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); + } + } + } +} diff --git a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs new file mode 100644 index 00000000..55789c23 --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs @@ -0,0 +1,69 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchResultManager : ISearchResultManager + { + public string Name => Constants.ModuleSearchIndexManagerName; + + private readonly IServiceProvider _serviceProvider; + + public ModuleSearchResultManager( + IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)) + { + var page = pageRepository.GetPage(pageId); + if (page != null) + { + return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + } + + return string.Empty; + } + + public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) + { + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + var moduleIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchModuleIdPropertyName)?.Value ?? string.Empty; + if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId) + && !string.IsNullOrEmpty(moduleIdValue) && int.TryParse(moduleIdValue, out int moduleId)) + { + return CanViewPage(pageId, searchQuery.User) && CanViewModule(moduleId, searchQuery.User); + } + + return false; + } + + private bool CanViewModule(int moduleId, User user) + { + var moduleRepository = _serviceProvider.GetRequiredService(); + var module = moduleRepository.GetModule(moduleId); + return module != null && !module.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, module.PermissionList); + } + + private bool CanViewPage(int pageId, User user) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(pageId); + + return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) + && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList)); + } + } +} diff --git a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs new file mode 100644 index 00000000..68c4af02 --- /dev/null +++ b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Mvc.RazorPages; +using System.Reflection.Metadata; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class PageSearchIndexManager : SearchIndexManagerBase + { + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageRepository _pageRepository; + + public PageSearchIndexManager( + IServiceProvider serviceProvider, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageRepository = pageRepository; + } + + public override string Name => Constants.PageSearchIndexManagerName; + + public override int Priority => Constants.PageSearchIndexManagerPriority; + + public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + { + var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue); + var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue); + var searchDocuments = new List(); + + foreach(var page in pages) + { + try + { + if(IsSystemPage(page)) + { + continue; + } + + var document = new SearchDocument + { + EntryId = page.PageId, + IndexerName = Name, + SiteId = page.SiteId, + LanguageCode = string.Empty, + ModifiedTime = page.ModifiedOn, + AdditionalContent = string.Empty, + Url = page.Url ?? string.Empty, + Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, + Description = string.Empty, + Body = $"{page.Name} {page.Title}" + }; + + if (document.Properties == null) + { + document.Properties = new List(); + } + + if (!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); + } + + searchDocuments.Add(document); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index page {page.PageId} failed."); + handleError($"Search: Index page {page.PageId} failed: {ex.Message}"); + } + } + + processSearchDocuments(searchDocuments); + + return searchDocuments.Count; + } + + private bool IsSystemPage(Models.Page page) + { + return page.Path.Contains("admin") || page.Path == "login" || page.Path == "register" || page.Path == "profile"; + } + } +} diff --git a/Oqtane.Server/Managers/Search/PageSearchResultManager.cs b/Oqtane.Server/Managers/Search/PageSearchResultManager.cs new file mode 100644 index 00000000..fd5456a4 --- /dev/null +++ b/Oqtane.Server/Managers/Search/PageSearchResultManager.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class PageSearchResultManager : ISearchResultManager + { + public string Name => Constants.PageSearchIndexManagerName; + + private readonly IServiceProvider _serviceProvider; + + public PageSearchResultManager( + IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(searchResult.EntryId); + if (page != null) + { + return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + + return string.Empty; + } + + public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(searchResult.EntryId); + + return page != null && !page.IsDeleted + && UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.View, page.PermissionList) + && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.Edit, page.PermissionList)); + } + } +} diff --git a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs new file mode 100644 index 00000000..0ecb2880 --- /dev/null +++ b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public abstract class SearchIndexManagerBase : ISearchIndexManager + { + private readonly IServiceProvider _serviceProvider; + + public SearchIndexManagerBase(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public abstract int Priority { get; } + + public abstract string Name { get; } + + public abstract int IndexDocuments(int siteId, DateTime? startDate, Action> processSearchDocuments, Action handleError); + + public virtual bool IsIndexEnabled(int siteId) + { + var settingName = string.Format(Constants.SearchIndexManagerEnabledSettingFormat, Name); + var settingRepository = _serviceProvider.GetRequiredService(); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName); + return setting == null || setting.SettingValue == "true"; + } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs new file mode 100644 index 00000000..2f5b6d8d --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs @@ -0,0 +1,64 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentEntityBuilder : AuditableBaseEntityBuilder + { + private const string _entityTableName = "SearchDocument"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocument", x => x.SearchDocumentId); + + public SearchDocumentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + } + + protected override SearchDocumentEntityBuilder BuildTable(ColumnsBuilder table) + { + SearchDocumentId = AddAutoIncrementColumn(table, "SearchDocumentId"); + EntryId = AddIntegerColumn(table, "EntryId"); + IndexerName = AddStringColumn(table, "IndexerName", 50); + SiteId = AddIntegerColumn(table, "SiteId"); + Title = AddStringColumn(table, "Title", 255); + Description = AddMaxStringColumn(table, "Description"); + Body = AddMaxStringColumn(table, "Body"); + Url = AddStringColumn(table, "Url", 255); + ModifiedTime = AddDateTimeColumn(table, "ModifiedTime"); + IsActive = AddBooleanColumn(table, "IsActive"); + AdditionalContent = AddMaxStringColumn(table, "AdditionalContent"); + LanguageCode = AddStringColumn(table, "LanguageCode", 20); + + AddAuditableColumns(table); + + return this; + } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder EntryId { get; private set; } + + public OperationBuilder IndexerName { get; private set; } + + public OperationBuilder SiteId { get; private set; } + + public OperationBuilder Title { get; private set; } + + public OperationBuilder Description { get; private set; } + + public OperationBuilder Body { get; private set; } + + public OperationBuilder Url { get; private set; } + + public OperationBuilder ModifiedTime { get; private set; } + + public OperationBuilder IsActive { get; private set; } + + public OperationBuilder AdditionalContent { get; private set; } + + public OperationBuilder LanguageCode { get; private set; } + + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs new file mode 100644 index 00000000..a591aec2 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentPropertyEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchDocumentProperty"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentProperty", x => x.PropertyId); + private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentProperty_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); + + public SearchDocumentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchDocumentForeignKey); + } + + protected override SearchDocumentPropertyEntityBuilder BuildTable(ColumnsBuilder table) + { + PropertyId = AddAutoIncrementColumn(table, "PropertyId"); + SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); + Name = AddStringColumn(table, "Name", 50); + Value = AddStringColumn(table, "Value", 50); + + return this; + } + + public OperationBuilder PropertyId { get; private set; } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder Name { get; private set; } + + public OperationBuilder Value { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs new file mode 100644 index 00000000..bd864f22 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs @@ -0,0 +1,37 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentTagEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchDocumentTag"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentTag", x => x.TagId); + private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentTag_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); + + public SearchDocumentTagEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchDocumentForeignKey); + } + + protected override SearchDocumentTagEntityBuilder BuildTable(ColumnsBuilder table) + { + TagId = AddAutoIncrementColumn(table, "TagId"); + SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); + Tag = AddStringColumn(table, "Tag", 50); + + return this; + } + + public OperationBuilder TagId { get; private set; } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder Tag { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs new file mode 100644 index 00000000..f32a8137 --- /dev/null +++ b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Oqtane.Databases.Interfaces; +using Oqtane.Migrations.EntityBuilders; +using Oqtane.Repository; + +namespace Oqtane.Migrations.Tenant +{ + [DbContext(typeof(TenantDBContext))] + [Migration("Tenant.05.02.00.01")] + public class AddSearchTables : MultiDatabaseMigration + { + public AddSearchTables(IDatabase database) : base(database) + { + } + + protected override void Up(MigrationBuilder migrationBuilder) + { + var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentEntityBuilder.Create(); + + var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentPropertyEntityBuilder.Create(); + + var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentTagEntityBuilder.Create(); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentPropertyEntityBuilder.Drop(); + + var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentTagEntityBuilder.Drop(); + + var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentEntityBuilder.Drop(); + } + } +} diff --git a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs index 6c9293fb..8cabf391 100644 --- a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs +++ b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs @@ -8,20 +8,29 @@ using Oqtane.Shared; using Oqtane.Migrations.Framework; using Oqtane.Documentation; using System.Linq; +using Oqtane.Interfaces; +using System.Collections.Generic; +using System; // ReSharper disable ConvertToUsingDeclaration namespace Oqtane.Modules.HtmlText.Manager { [PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")] - public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable + public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, IModuleSearch { + private readonly IServiceProvider _serviceProvider; private readonly IHtmlTextRepository _htmlText; private readonly IDBContextDependencies _DBContextDependencies; private readonly ISqlRepository _sqlRepository; - public HtmlTextManager(IHtmlTextRepository htmlText, IDBContextDependencies DBContextDependencies, ISqlRepository sqlRepository) + public HtmlTextManager( + IServiceProvider serviceProvider, + IHtmlTextRepository htmlText, + IDBContextDependencies DBContextDependencies, + ISqlRepository sqlRepository) { + _serviceProvider = serviceProvider; _htmlText = htmlText; _DBContextDependencies = DBContextDependencies; _sqlRepository = sqlRepository; @@ -39,6 +48,27 @@ namespace Oqtane.Modules.HtmlText.Manager return content; } + public IList GetSearchDocuments(Module module, DateTime startDate) + { + var searchDocuments = new List(); + + var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId); + if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate)) + { + var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First(); + + searchDocuments.Add(new SearchDocument + { + Title = module.Title, + Description = string.Empty, + Body = SearchUtils.Clean(htmltext.Content, true), + ModifiedTime = htmltext.ModifiedOn + }); + } + + return searchDocuments; + } + public void ImportModule(Module module, string content, string version) { content = WebUtility.HtmlDecode(content); diff --git a/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs b/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs new file mode 100644 index 00000000..d134b1ae --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs @@ -0,0 +1,43 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Oqtane.Controllers; +using Oqtane.Documentation; +using Oqtane.Enums; +using Oqtane.Infrastructure; +using Oqtane.Modules.SearchResults.Services; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults.Controllers +{ + [Route(ControllerRoutes.ApiRoute)] + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class SearchResultsController : ModuleControllerBase + { + private readonly ISearchResultsService _searchResultsService; + + public SearchResultsController(ISearchResultsService searchResultsService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor) + { + _searchResultsService = searchResultsService; + } + + [HttpPost] + [Authorize(Policy = PolicyNames.ViewModule)] + public async Task Post([FromBody] Models.SearchQuery searchQuery) + { + try + { + return await _searchResultsService.SearchAsync(AuthEntityId(EntityNames.Module), searchQuery); + } + catch(Exception ex) + { + _logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery); + HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return null; + } + } + } +} diff --git a/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs new file mode 100644 index 00000000..3823bac6 --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs @@ -0,0 +1,36 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Oqtane.Documentation; +using Oqtane.Infrastructure; +using Oqtane.Models; +using Oqtane.Services; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class ServerSearchResultsService : ISearchResultsService, ITransientService + { + private readonly ILogManager _logger; + private readonly IHttpContextAccessor _accessor; + private readonly Alias _alias; + private readonly ISearchService _searchService; + + public ServerSearchResultsService( + ITenantManager tenantManager, + ILogManager logger, + IHttpContextAccessor accessor, + ISearchService searchService) + { + _logger = logger; + _accessor = accessor; + _alias = tenantManager.GetAlias(); + _searchService = searchService; + } + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + var results = await _searchService.SearchAsync(searchQuery); + return results; + } + } +} diff --git a/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs b/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs new file mode 100644 index 00000000..25d2d15f --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Infrastructure; +using Oqtane.Modules.SearchResults.Services; + +namespace Oqtane.Modules.SearchResults.Startup +{ + public class ServerStartup : IServerStartup + { + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + } + + public void ConfigureMvc(IMvcBuilder mvcBuilder) + { + } + + public void ConfigureServices(IServiceCollection services) + { + services.AddTransient(); + } + } +} diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs new file mode 100644 index 00000000..ac2ed576 --- /dev/null +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; +using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; + +namespace Oqtane.Providers +{ + public class DatabaseSearchProvider : ISearchProvider + { + private readonly ISearchDocumentRepository _searchDocumentRepository; + + private const float TitleBoost = 100f; + private const float DescriptionBoost = 10f; + private const float BodyBoost = 10f; + private const float AdditionalContentBoost = 5f; + + public string Name => Constants.DefaultSearchProviderName; + + public DatabaseSearchProvider(ISearchDocumentRepository searchDocumentRepository) + { + _searchDocumentRepository = searchDocumentRepository; + } + + public void Commit() + { + } + + public void DeleteDocument(string id) + { + _searchDocumentRepository.DeleteSearchDocument(id); + } + + public bool Optimize() + { + return true; + } + + public void ResetIndex() + { + _searchDocumentRepository.DeleteAllSearchDocuments(); + } + + public void SaveDocument(SearchDocument document, bool autoCommit = false) + { + //remove exist document + _searchDocumentRepository.DeleteSearchDocument(document.IndexerName, document.EntryId); + + _searchDocumentRepository.AddSearchDocument(document); + } + + public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) + { + var totalResults = 0; + + var documents = await _searchDocumentRepository.GetSearchDocumentsAsync(searchQuery); + + //convert the search documents to search results. + var results = documents + .Where(i => validateFunc(i, searchQuery)) + .Select(i => ConvertToSearchResult(i, searchQuery)); + + if (searchQuery.SortDirection == SearchSortDirections.Descending) + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderByDescending(i => i.ModifiedTime); + break; + } + } + else + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderBy(i => i.ModifiedTime); + break; + } + } + + //remove duplicated results based on page id for Page and Module types + results = results.DistinctBy(i => + { + if (i.IndexerName == Constants.PageSearchIndexManagerName || i.IndexerName == Constants.ModuleSearchIndexManagerName) + { + var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey; + } + else + { + return i.UniqueKey; + } + }); + + totalResults = results.Count(); + + return new SearchResults + { + Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(), + TotalResults = totalResults + }; + } + + private SearchResult ConvertToSearchResult(SearchDocument searchDocument, SearchQuery searchQuery) + { + var searchResult = new SearchResult() + { + SearchDocumentId = searchDocument.SearchDocumentId, + SiteId = searchDocument.SiteId, + IndexerName = searchDocument.IndexerName, + EntryId = searchDocument.EntryId, + Title = searchDocument.Title, + Description = searchDocument.Description, + Body = searchDocument.Body, + Url = searchDocument.Url, + ModifiedTime = searchDocument.ModifiedTime, + Tags = searchDocument.Tags, + Properties = searchDocument.Properties, + Snippet = BuildSnippet(searchDocument, searchQuery), + Score = CalculateScore(searchDocument, searchQuery) + }; + + return searchResult; + } + + private float CalculateScore(SearchDocument searchDocument, SearchQuery searchQuery) + { + var score = 0f; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + score += Regex.Matches(searchDocument.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost; + score += Regex.Matches(searchDocument.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost; + score += Regex.Matches(searchDocument.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost; + score += Regex.Matches(searchDocument.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost; + } + + return score / 100; + } + + private string BuildSnippet(SearchDocument searchDocument, SearchQuery searchQuery) + { + var content = $"{searchDocument.Title} {searchDocument.Description} {searchDocument.Body}"; + var snippet = string.Empty; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + if (!string.IsNullOrWhiteSpace(keyword) && content.Contains(keyword, StringComparison.OrdinalIgnoreCase)) + { + var start = content.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) - 20; + var prefix = "..."; + var suffix = "..."; + if (start <= 0) + { + start = 0; + prefix = string.Empty; + } + + var length = searchQuery.BodySnippetLength; + if (start + length >= content.Length) + { + length = content.Length - start; + suffix = string.Empty; + } + + snippet = $"{prefix}{content.Substring(start, length)}{suffix}"; + break; + + + } + } + + if (string.IsNullOrEmpty(snippet)) + { + snippet = content.Substring(0, searchQuery.BodySnippetLength); + } + + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + snippet = Regex.Replace(snippet, $"({keyword})", $"$1", RegexOptions.IgnoreCase); + } + + return snippet; + } + } +} diff --git a/Oqtane.Server/Repository/Context/TenantDBContext.cs b/Oqtane.Server/Repository/Context/TenantDBContext.cs index 2efad3f7..cbe57631 100644 --- a/Oqtane.Server/Repository/Context/TenantDBContext.cs +++ b/Oqtane.Server/Repository/Context/TenantDBContext.cs @@ -29,5 +29,8 @@ namespace Oqtane.Repository public virtual DbSet Language { get; set; } public virtual DbSet Visitor { get; set; } public virtual DbSet UrlMapping { get; set; } + public virtual DbSet SearchDocument { get; set; } + public virtual DbSet SearchDocumentProperty { get; set; } + public virtual DbSet SearchDocumentTag { get; set; } } } diff --git a/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs b/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs new file mode 100644 index 00000000..6128c40c --- /dev/null +++ b/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Repository +{ + public interface ISearchDocumentRepository + { + Task> GetSearchDocumentsAsync(SearchQuery searchQuery); + SearchDocument AddSearchDocument(SearchDocument searchDocument); + void DeleteSearchDocument(int searchDocumentId); + void DeleteSearchDocument(string indexerName, int entryId); + void DeleteSearchDocument(string uniqueKey); + void DeleteAllSearchDocuments(); + } +} diff --git a/Oqtane.Server/Repository/SearchDocumentRepository.cs b/Oqtane.Server/Repository/SearchDocumentRepository.cs new file mode 100644 index 00000000..bdceb52e --- /dev/null +++ b/Oqtane.Server/Repository/SearchDocumentRepository.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Repository +{ + public class SearchDocumentRepository : ISearchDocumentRepository + { + private readonly IDbContextFactory _dbContextFactory; + + public SearchDocumentRepository(IDbContextFactory dbContextFactory) + { + _dbContextFactory = dbContextFactory; + } + + public async Task> GetSearchDocumentsAsync(SearchQuery searchQuery) + { + using var db = _dbContextFactory.CreateDbContext(); + var documents = db.SearchDocument.AsNoTracking() + .Include(i => i.Properties) + .Include(i => i.Tags) + .Where(i => i.SiteId == searchQuery.SiteId); + + if (searchQuery.Sources != null && searchQuery.Sources.Any()) + { + documents = documents.Where(i => searchQuery.Sources.Contains(i.IndexerName)); + } + + if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue) + { + documents = documents.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc); + } + + if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue) + { + documents = documents.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc); + } + + if (searchQuery.Tags != null && searchQuery.Tags.Any()) + { + foreach (var tag in searchQuery.Tags) + { + documents = documents.Where(i => i.Tags.Any(t => t.Tag == tag)); + } + } + + if (searchQuery.Properties != null && searchQuery.Properties.Any()) + { + foreach (var property in searchQuery.Properties) + { + documents = documents.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value)); + } + } + + var filteredDocuments = new List(); + if (!string.IsNullOrEmpty(searchQuery.Keywords)) + { + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + filteredDocuments.AddRange(await documents.Where(i => i.Title.Contains(keyword) || i.Description.Contains(keyword) || i.Body.Contains(keyword)).ToListAsync()); + } + } + + return filteredDocuments.DistinctBy(i => i.UniqueKey); + } + + public SearchDocument AddSearchDocument(SearchDocument searchDocument) + { + using var context = _dbContextFactory.CreateDbContext(); + context.SearchDocument.Add(searchDocument); + + if(searchDocument.Properties != null && searchDocument.Properties.Any()) + { + foreach(var property in searchDocument.Properties) + { + property.SearchDocumentId = searchDocument.SearchDocumentId; + context.SearchDocumentProperty.Add(property); + } + } + + if (searchDocument.Tags != null && searchDocument.Tags.Any()) + { + foreach (var tag in searchDocument.Tags) + { + tag.SearchDocumentId = searchDocument.SearchDocumentId; + context.SearchDocumentTag.Add(tag); + } + } + + context.SaveChanges(); + + return searchDocument; + } + + public void DeleteSearchDocument(int searchDocumentId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.Find(searchDocumentId); + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + + public void DeleteSearchDocument(string indexerName, int entryId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.FirstOrDefault(i => i.IndexerName == indexerName && i.EntryId == entryId); + if(searchDocument != null) + { + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + } + + public void DeleteSearchDocument(string uniqueKey) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.FirstOrDefault(i => (i.IndexerName + ":" + i.EntryId) == uniqueKey); + if (searchDocument != null) + { + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + } + + public void DeleteAllSearchDocuments() + { + using var db = _dbContextFactory.CreateDbContext(); + db.SearchDocument.RemoveRange(db.SearchDocument); + db.SaveChanges(); + } + } +} diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs new file mode 100644 index 00000000..caa9fce6 --- /dev/null +++ b/Oqtane.Server/Services/SearchService.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Infrastructure; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + public class SearchService : ISearchService + { + private readonly IServiceProvider _serviceProvider; + private readonly ITenantManager _tenantManager; + private readonly IAliasRepository _aliasRepository; + private readonly ISettingRepository _settingRepository; + private readonly ILogger _logger; + private readonly IMemoryCache _cache; + + public SearchService( + IServiceProvider serviceProvider, + ITenantManager tenantManager, + IAliasRepository aliasRepository, + ISettingRepository settingRepository, + ILogger logger, + IMemoryCache cache) + { + _tenantManager = tenantManager; + _aliasRepository = aliasRepository; + _settingRepository = settingRepository; + _serviceProvider = serviceProvider; + _logger = logger; + _cache = cache; + } + + public void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError) + { + var searchEnabled = SearchEnabled(siteId); + if(!searchEnabled) + { + logNote($"Search: Search is disabled on site {siteId}.
"); + return; + } + + _logger.LogDebug($"Search: Start Index Content of {siteId}, Start Time: {startTime.GetValueOrDefault(DateTime.MinValue)}"); + + var searchProvider = GetSearchProvider(siteId); + + SetTenant(siteId); + + if (startTime == null) + { + searchProvider.ResetIndex(); + } + + var searchIndexManagers = GetSearchIndexManagers(m => { }); + foreach (var searchIndexManager in searchIndexManagers) + { + if (!searchIndexManager.IsIndexEnabled(siteId)) + { + logNote($"Search: Ignore indexer {searchIndexManager.Name} because it's disabled.
"); + } + else + { + _logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}"); + + var count = searchIndexManager.IndexDocuments(siteId, startTime, SaveIndexDocuments, handleError); + logNote($"Search: Indexer {searchIndexManager.Name} processed {count} documents.
"); + + _logger.LogDebug($"Search: End Index {searchIndexManager.Name}"); + } + } + } + + public async Task SearchAsync(SearchQuery searchQuery) + { + var searchProvider = GetSearchProvider(searchQuery.SiteId); + var searchResults = await searchProvider.SearchAsync(searchQuery, HasViewPermission); + + //generate the document url if it's not set. + foreach (var result in searchResults.Results) + { + if(string.IsNullOrEmpty(result.Url)) + { + result.Url = GetDocumentUrl(result, searchQuery); + } + } + + return searchResults; + } + + private ISearchProvider GetSearchProvider(int siteId) + { + var providerName = GetSearchProviderSetting(siteId); + var searchProviders = _serviceProvider.GetServices(); + var provider = searchProviders.FirstOrDefault(i => i.Name == providerName); + if(provider == null) + { + provider = searchProviders.FirstOrDefault(i => i.Name == Constants.DefaultSearchProviderName); + } + + return provider; + } + + private string GetSearchProviderSetting(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchProviderSettingName); + if(!string.IsNullOrEmpty(setting?.SettingValue)) + { + return setting.SettingValue; + } + + return Constants.DefaultSearchProviderName; + } + + private bool SearchEnabled(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchEnabledSettingName); + if (!string.IsNullOrEmpty(setting?.SettingValue)) + { + return bool.TryParse(setting.SettingValue, out bool enabled) && enabled; + } + + return true; + } + + private void SetTenant(int siteId) + { + var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId); + _tenantManager.SetAlias(alias); + } + + private IList GetSearchIndexManagers(Action initManager) + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchIndexManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchIndexManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + initManager(manager); + managers.Add(manager); + } + + return managers.OrderBy(i => i.Priority).ToList(); + } + + private IList GetSearchResultManagers() + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + managers.Add(manager); + } + + return managers.ToList(); + } + + private void SaveIndexDocuments(IList searchDocuments) + { + if(searchDocuments.Any()) + { + var searchProvider = GetSearchProvider(searchDocuments.First().SiteId); + + foreach (var searchDocument in searchDocuments) + { + try + { + searchProvider.SaveDocument(searchDocument); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Save search document {searchDocument.UniqueKey} failed."); + } + } + + //commit the index changes + searchProvider.Commit(); + } + } + + private bool HasViewPermission(SearchDocument searchDocument, SearchQuery searchQuery) + { + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchDocument.IndexerName); + if (searchResultManager != null) + { + return searchResultManager.Visible(searchDocument, searchQuery); + } + return true; + } + + private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery) + { + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.IndexerName); + if(searchResultManager != null) + { + return searchResultManager.GetUrl(result, searchQuery); + } + + return string.Empty; + } + } +} diff --git a/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css new file mode 100644 index 00000000..71194e38 --- /dev/null +++ b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css @@ -0,0 +1,3 @@ +.search-result-container ul.pagination li label, .search-result-container ul.dropdown-menu li label { + cursor: pointer; +} \ No newline at end of file diff --git a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css index 1b9ea409..bbb65908 100644 --- a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css +++ b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css @@ -79,6 +79,10 @@ body { top: -2px; } +.app-search input{ + width: auto; +} + .navbar-toggler { background-color: rgba(255, 255, 255, 0.1); margin: .5rem; diff --git a/Oqtane.Server/wwwroot/css/app.css b/Oqtane.Server/wwwroot/css/app.css index f5a65a80..941ff5f1 100644 --- a/Oqtane.Server/wwwroot/css/app.css +++ b/Oqtane.Server/wwwroot/css/app.css @@ -235,3 +235,17 @@ app { .app-form-inline { display: inline-block; } +.app-search{ + display: inline-block; + position: relative; +} +.app-search input + button{ + background: none; + border: none; + position: absolute; + right: 0; + top: 0; +} +.app-search input + button .oi{ + top: 0; +} \ No newline at end of file diff --git a/Oqtane.Shared/Enums/SearchSortDirections.cs b/Oqtane.Shared/Enums/SearchSortDirections.cs new file mode 100644 index 00000000..62c7f08d --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortDirections.cs @@ -0,0 +1,10 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortDirections + { + Ascending, + Descending + } +} diff --git a/Oqtane.Shared/Enums/SearchSortFields.cs b/Oqtane.Shared/Enums/SearchSortFields.cs new file mode 100644 index 00000000..c8e16175 --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortFields.cs @@ -0,0 +1,11 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortFields + { + Relevance, + Title, + LastModified + } +} diff --git a/Oqtane.Shared/Interfaces/IModuleSearch.cs b/Oqtane.Shared/Interfaces/IModuleSearch.cs new file mode 100644 index 00000000..af94a300 --- /dev/null +++ b/Oqtane.Shared/Interfaces/IModuleSearch.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Interfaces +{ + public interface IModuleSearch + { + public IList GetSearchDocuments(Module module, DateTime startTime); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs new file mode 100644 index 00000000..9a322ab5 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Web; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchIndexManager + { + int Priority { get; } + + string Name { get; } + + bool IsIndexEnabled(int siteId); + + int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchProvider.cs b/Oqtane.Shared/Interfaces/ISearchProvider.cs new file mode 100644 index 00000000..e1e9fb16 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchProvider.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchProvider + { + string Name { get; } + + void SaveDocument(SearchDocument document, bool autoCommit = false); + + void DeleteDocument(string id); + + Task SearchAsync(SearchQuery searchQuery, Func validateFunc); + + bool Optimize(); + + void Commit(); + + void ResetIndex(); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchResultManager.cs b/Oqtane.Shared/Interfaces/ISearchResultManager.cs new file mode 100644 index 00000000..2dc459b0 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchResultManager.cs @@ -0,0 +1,14 @@ +using System; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchResultManager + { + string Name { get; } + + bool Visible(SearchDocument searchResult, SearchQuery searchQuery); + + string GetUrl(SearchResult searchResult, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchService.cs b/Oqtane.Shared/Interfaces/ISearchService.cs new file mode 100644 index 00000000..0f125c65 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchService.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchService + { + void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError); + + Task SearchAsync(SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Models/SearchDocument.cs b/Oqtane.Shared/Models/SearchDocument.cs new file mode 100644 index 00000000..67161b3b --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocument.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Text.Json; +namespace Oqtane.Models +{ + public class SearchDocument : ModelBase + { + public int SearchDocumentId { get; set; } + + [NotMapped] + public string UniqueKey => $"{IndexerName}:{EntryId}"; + + public int EntryId { get; set; } + + public string IndexerName { get; set; } + + public int SiteId { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Body { get; set; } + + public string Url { get; set; } + + public DateTime ModifiedTime { get; set; } + + public bool IsActive { get; set; } + + public string AdditionalContent { get; set; } + + public string LanguageCode { get; set; } + + public IList Tags { get; set; } + + public IList Properties { get; set; } + + public override string ToString() + { + return JsonSerializer.Serialize(this); + } + } +} diff --git a/Oqtane.Shared/Models/SearchDocumentProperty.cs b/Oqtane.Shared/Models/SearchDocumentProperty.cs new file mode 100644 index 00000000..4febae43 --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocumentProperty.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.EntityFrameworkCore; + +namespace Oqtane.Models +{ + public class SearchDocumentProperty + { + [Key] + public int PropertyId { get; set; } + + public int SearchDocumentId { get; set; } + + public string Name { get; set; } + + public string Value { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchDocumentTag.cs b/Oqtane.Shared/Models/SearchDocumentTag.cs new file mode 100644 index 00000000..ee929efc --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocumentTag.cs @@ -0,0 +1,14 @@ +using System.ComponentModel.DataAnnotations; + +namespace Oqtane.Models +{ + public class SearchDocumentTag + { + [Key] + public int TagId { get; set; } + + public int SearchDocumentId { get; set; } + + public string Tag { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchQuery.cs b/Oqtane.Shared/Models/SearchQuery.cs new file mode 100644 index 00000000..f148252d --- /dev/null +++ b/Oqtane.Shared/Models/SearchQuery.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using Oqtane.Shared; + +namespace Oqtane.Models +{ + public class SearchQuery + { + public int SiteId { get; set; } + + public Alias Alias { get; set; } + + public User User { get; set; } + + public string Keywords { get; set; } + + public IList Sources { get; set; } = new List(); + + public DateTime BeginModifiedTimeUtc { get; set; } + + public DateTime EndModifiedTimeUtc { get; set; } + + public IList Tags { get; set; } = new List(); + + public IDictionary Properties { get; set; } = new Dictionary(); + + public int PageIndex { get; set; } + + public int PageSize { get; set; } + + public SearchSortFields SortField { get; set; } + + public SearchSortDirections SortDirection { get; set; } + + public int BodySnippetLength { get; set;} = 255; + } +} diff --git a/Oqtane.Shared/Models/SearchResult.cs b/Oqtane.Shared/Models/SearchResult.cs new file mode 100644 index 00000000..d13cafa1 --- /dev/null +++ b/Oqtane.Shared/Models/SearchResult.cs @@ -0,0 +1,11 @@ +namespace Oqtane.Models +{ + public class SearchResult : SearchDocument + { + public float Score { get; set; } + + public string DisplayScore { get; set; } + + public string Snippet { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchResults.cs b/Oqtane.Shared/Models/SearchResults.cs new file mode 100644 index 00000000..5642468c --- /dev/null +++ b/Oqtane.Shared/Models/SearchResults.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Oqtane.Models +{ + public class SearchResults + { + public IList Results { get; set; } + + public int TotalResults { get; set; } + } +} diff --git a/Oqtane.Shared/Shared/Constants.cs b/Oqtane.Shared/Shared/Constants.cs index b2b8c837..f1a14cd4 100644 --- a/Oqtane.Shared/Shared/Constants.cs +++ b/Oqtane.Shared/Shared/Constants.cs @@ -77,6 +77,22 @@ namespace Oqtane.Shared public static readonly string VisitorCookiePrefix = "APP_VISITOR_"; + public const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; + public const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; + public const string SearchResultManagersCacheName = "SearchResultManagers"; + public const int SearchDefaultPageSize = 10; + public const string SearchPageIdPropertyName = "PageId"; + public const string SearchModuleIdPropertyName = "ModuleId"; + public const string DefaultSearchProviderName = "Database"; + public const string SearchProviderSettingName = "SearchProvider"; + public const string SearchEnabledSettingName = "SearchEnabled"; + + public const string ModuleSearchIndexManagerName = "Module"; + public const string PageSearchIndexManagerName = "Page"; + + public const int PageSearchIndexManagerPriority = 100; + public const int ModuleSearchIndexManagerPriority = 200; + // Obsolete constants const string RoleObsoleteMessage = "Use the corresponding member from Oqtane.Shared.RoleNames"; diff --git a/Oqtane.Shared/Shared/SearchUtils.cs b/Oqtane.Shared/Shared/SearchUtils.cs new file mode 100644 index 00000000..b7116d4a --- /dev/null +++ b/Oqtane.Shared/Shared/SearchUtils.cs @@ -0,0 +1,95 @@ +using System.Collections; +using System.Collections.Generic; +using System.Net; +using System.Text.RegularExpressions; + +namespace Oqtane.Shared +{ + public sealed class SearchUtils + { + private const string PunctuationMatch = "[~!#\\$%\\^&*\\(\\)-+=\\{\\[\\}\\]\\|;:\\x22'<,>\\.\\?\\\\\\t\\r\\v\\f\\n]"; + private static readonly Regex _stripWhiteSpaceRegex = new Regex("\\s+", RegexOptions.Compiled); + private static readonly Regex _stripTagsRegex = new Regex("<[^<>]*>", RegexOptions.Compiled); + private static readonly Regex _afterRegEx = new Regex(PunctuationMatch + "\\s", RegexOptions.Compiled); + private static readonly Regex _beforeRegEx = new Regex("\\s" + PunctuationMatch, RegexOptions.Compiled); + + public static string Clean(string html, bool removePunctuation) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + if (html.Contains("<")) + { + html = WebUtility.HtmlDecode(html); + } + + html = StripTags(html, true); + html = WebUtility.HtmlDecode(html); + + if (removePunctuation) + { + html = StripPunctuation(html, true); + html = StripWhiteSpace(html, true); + } + + return html; + } + + public static IList GetKeywordsList(string keywords) + { + var keywordsList = new List(); + if(!string.IsNullOrEmpty(keywords)) + { + foreach (var keyword in keywords.Split(' ')) + { + if (!string.IsNullOrWhiteSpace(keyword.Trim())) + { + keywordsList.Add(keyword.Trim()); + } + } + } + + return keywordsList; + } + + private static string StripTags(string html, bool retainSpace) + { + return _stripTagsRegex.Replace(html, retainSpace ? " " : string.Empty); + } + + private static string StripPunctuation(string html, bool retainSpace) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + string retHTML = html + " "; + + var repString = retainSpace ? " " : string.Empty; + while (_beforeRegEx.IsMatch(retHTML)) + { + retHTML = _beforeRegEx.Replace(retHTML, repString); + } + + while (_afterRegEx.IsMatch(retHTML)) + { + retHTML = _afterRegEx.Replace(retHTML, repString); + } + + return retHTML.Trim('"'); + } + + private static string StripWhiteSpace(string html, bool retainSpace) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + return _stripWhiteSpaceRegex.Replace(html, retainSpace ? " " : string.Empty); + } + } +} From 7f970d489f414ef99e2d35b788f2794348fc786b Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 4 Jun 2024 17:32:31 +0800 Subject: [PATCH 2/5] refactoring the code. --- .../{ => Admin}/SearchResults/Index.razor | 66 +++---- .../{ => Admin}/SearchResults/ModuleInfo.cs | 9 +- .../{ => Admin}/SearchResults/Settings.razor | 10 +- .../Services/SearchResultsService.cs | 23 --- .../{ => Admin}/SearchResults/Index.resx | 0 .../{ => Admin}/SearchResults/Settings.resx | 0 .../Interfaces}/ISearchResultsService.cs | 4 +- .../Services/SearchResultsService.cs | 23 +++ .../Themes/Controls/Theme/Search.razor | 6 +- .../Controllers/SearchResultsController.cs | 16 +- .../OqtaneServiceCollectionExtensions.cs | 3 +- .../Infrastructure/Jobs/SearchIndexJob.cs | 8 +- .../SiteTemplates/DefaultSiteTemplate.cs | 4 +- .../Infrastructure/UpgradeManager.cs | 46 +++++ .../Search/ModuleSearchIndexManager.cs | 85 +++++---- .../Search/ModuleSearchResultManager.cs | 17 +- .../Managers/Search/PageSearchIndexManager.cs | 43 +++-- .../Search/PageSearchResultManager.cs | 46 ----- .../Managers/Search/SearchIndexManagerBase.cs | 6 +- ...ilder.cs => SearchContentEntityBuilder.cs} | 27 ++- .../SearchContentPropertyEntityBuilder.cs | 40 +++++ .../SearchContentWordCountEntityBuilder.cs | 42 +++++ .../SearchContentWordSourceEntityBuilder.cs | 31 ++++ .../SearchDocumentPropertyEntityBuilder.cs | 40 ----- .../SearchDocumentTagEntityBuilder.cs | 37 ---- .../Tenant/05020001_AddSearchTables.cs | 32 ++-- .../HtmlText/Manager/HtmlTextManager.cs | 12 +- .../Services/SearchResultsService.cs | 36 ---- .../SearchResults/Startup/ServerStartup.cs | 24 --- Oqtane.Server/Oqtane.Server.csproj | 1 + .../Providers/DatabaseSearchProvider.cs | 165 +++++++++++++---- .../Repository/Context/TenantDBContext.cs | 7 +- .../Interfaces/ISearchContentRepository.cs | 24 +++ .../Interfaces/ISearchDocumentRepository.cs | 17 -- .../Repository/SearchContentRepository.cs | 167 ++++++++++++++++++ .../Repository/SearchDocumentRepository.cs | 136 -------------- Oqtane.Server/Services/SearchService.cs | 81 +++++++-- .../Module.css | 0 .../Interfaces/ISearchIndexManager.cs | 2 +- Oqtane.Shared/Interfaces/ISearchProvider.cs | 6 +- .../Interfaces/ISearchResultManager.cs | 2 +- .../{IModuleSearch.cs => ISearchable.cs} | 4 +- .../{SearchDocument.cs => SearchContent.cs} | 18 +- ...ntProperty.cs => SearchContentProperty.cs} | 4 +- .../Models/SearchContentWordSource.cs | 12 ++ Oqtane.Shared/Models/SearchContentWords.cs | 19 ++ Oqtane.Shared/Models/SearchDocumentTag.cs | 14 -- Oqtane.Shared/Models/SearchQuery.cs | 4 +- Oqtane.Shared/Models/SearchResult.cs | 2 +- Oqtane.Shared/Shared/Constants.cs | 16 +- Oqtane.Shared/Shared/SearchUtils.cs | 69 +------- 51 files changed, 806 insertions(+), 700 deletions(-) rename Oqtane.Client/Modules/{ => Admin}/SearchResults/Index.razor (68%) rename Oqtane.Client/Modules/{ => Admin}/SearchResults/ModuleInfo.cs (64%) rename Oqtane.Client/Modules/{ => Admin}/SearchResults/Settings.razor (83%) delete mode 100644 Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs rename Oqtane.Client/Resources/Modules/{ => Admin}/SearchResults/Index.resx (100%) rename Oqtane.Client/Resources/Modules/{ => Admin}/SearchResults/Settings.resx (100%) rename Oqtane.Client/{Modules/SearchResults/Services => Services/Interfaces}/ISearchResultsService.cs (67%) create mode 100644 Oqtane.Client/Services/SearchResultsService.cs rename Oqtane.Server/{Modules/SearchResults => }/Controllers/SearchResultsController.cs (57%) delete mode 100644 Oqtane.Server/Managers/Search/PageSearchResultManager.cs rename Oqtane.Server/Migrations/EntityBuilders/{SearchDocumentEntityBuilder.cs => SearchContentEntityBuilder.cs} (58%) create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs delete mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs delete mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs delete mode 100644 Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs delete mode 100644 Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs create mode 100644 Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs delete mode 100644 Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs create mode 100644 Oqtane.Server/Repository/SearchContentRepository.cs delete mode 100644 Oqtane.Server/Repository/SearchDocumentRepository.cs rename Oqtane.Server/wwwroot/Modules/{Oqtane.Modules.SearchResults => Oqtane.Modules.Admin.SearchResults}/Module.css (100%) rename Oqtane.Shared/Interfaces/{IModuleSearch.cs => ISearchable.cs} (58%) rename Oqtane.Shared/Models/{SearchDocument.cs => SearchContent.cs} (57%) rename Oqtane.Shared/Models/{SearchDocumentProperty.cs => SearchContentProperty.cs} (74%) create mode 100644 Oqtane.Shared/Models/SearchContentWordSource.cs create mode 100644 Oqtane.Shared/Models/SearchContentWords.cs delete mode 100644 Oqtane.Shared/Models/SearchDocumentTag.cs diff --git a/Oqtane.Client/Modules/SearchResults/Index.razor b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor similarity index 68% rename from Oqtane.Client/Modules/SearchResults/Index.razor rename to Oqtane.Client/Modules/Admin/SearchResults/Index.razor index 8ae0c087..2cbc3098 100644 --- a/Oqtane.Client/Modules/SearchResults/Index.razor +++ b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor @@ -1,22 +1,26 @@ -@using Oqtane.Modules.SearchResults.Services -@namespace Oqtane.Modules.SearchResults +@using Microsoft.AspNetCore.Http +@using Oqtane.Services +@using System.Net +@namespace Oqtane.Modules.Admin.SearchResults @inherits ModuleBase @inject ISearchResultsService SearchResultsService @inject IStringLocalizer Localizer +@inject IHttpContextAccessor HttpContext
-
- @Localizer["SearchPrefix"] - - -
+
+
+ @Localizer["SearchPrefix"] + + + +
+
@@ -35,12 +39,13 @@ Format="Grid" PageSize="@_pageSize.ToString()" DisplayPages="@_displayPages.ToString()" - CurrentPage="@_currentPage.ToString()" Columns="1" - Toolbar="Bottom"> + CurrentPage="@_currentPage.ToString()" + Columns="1" + Toolbar="Bottom" + Parameters="@($"q={_keywords}")">

@context.Title

-
@context.Url

@((MarkupString)context.Snippet)

 @@ -59,13 +64,16 @@
@code { + public override string RenderMode => RenderModes.Static; + private const int SearchDefaultPageSize = 10; + private SearchSortDirections _searchSortDirection = SearchSortDirections.Descending; //default sort by private SearchSortFields _searchSortField = SearchSortFields.Relevance; private string _keywords; private bool _loading; private SearchResults _searchResults; private int _currentPage = 0; - private int _pageSize = Constants.SearchDefaultPageSize; + private int _pageSize = SearchDefaultPageSize; private int _displayPages = 7; protected override async Task OnInitializedAsync() @@ -75,18 +83,9 @@ _pageSize = int.Parse(ModuleState.Settings["PageSize"]); } - if (PageState.QueryString.ContainsKey("s")) + if (PageState.QueryString.ContainsKey("q")) { - _keywords = PageState.QueryString["s"]; - } - - if (PageState.QueryString.ContainsKey("p")) - { - _currentPage = Convert.ToInt32(PageState.QueryString["p"]); - if (_currentPage < 1) - { - _currentPage = 1; - } + _keywords = WebUtility.UrlDecode(PageState.QueryString["q"]); } if (!string.IsNullOrEmpty(_keywords)) @@ -95,19 +94,9 @@ } } - private async Task KeywordsChanged(KeyboardEventArgs e) - { - if (e.Code == "Enter" || e.Code == "NumpadEnter") - { - if (!string.IsNullOrEmpty(_keywords)) - { - await Search(); - } - } - } - private async Task Search() { + _keywords = HttpContext.HttpContext.Request.Form["keywords"]; if (string.IsNullOrEmpty(_keywords)) { AddModuleMessage(Localizer["MissingKeywords"], MessageType.Warning); @@ -116,7 +105,6 @@ { ClearModuleMessage(); - _currentPage = 0; await PerformSearch(); } diff --git a/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs b/Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs similarity index 64% rename from Oqtane.Client/Modules/SearchResults/ModuleInfo.cs rename to Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs index ae4bc7d2..97a6835e 100644 --- a/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs +++ b/Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs @@ -3,19 +3,18 @@ using Oqtane.Documentation; using Oqtane.Models; using Oqtane.Shared; -namespace Oqtane.Modules.SearchResults +namespace Oqtane.Modules.Admin.SearchResults { - [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + [PrivateApi("Mark this as private, since it's not very useful in the public docs")] public class ModuleInfo : IModule { public ModuleDefinition ModuleDefinition => new ModuleDefinition { Name = "Search Results", Description = "Display Search Results", - Version = "1.0.0", + Version = Constants.Version, ServerManagerType = "", - ReleaseVersions = "1.0.0", - SettingsType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client", + SettingsType = "Oqtane.Modules.Admin.SearchResults.Settings, Oqtane.Client", Resources = new List() { new Resource { ResourceType = ResourceType.Stylesheet, Url = "~/Module.css" } diff --git a/Oqtane.Client/Modules/SearchResults/Settings.razor b/Oqtane.Client/Modules/Admin/SearchResults/Settings.razor similarity index 83% rename from Oqtane.Client/Modules/SearchResults/Settings.razor rename to Oqtane.Client/Modules/Admin/SearchResults/Settings.razor index b38d2946..1e98f0f6 100644 --- a/Oqtane.Client/Modules/SearchResults/Settings.razor +++ b/Oqtane.Client/Modules/Admin/SearchResults/Settings.razor @@ -1,7 +1,7 @@ -@namespace Oqtane.Modules.SearchResults +@namespace Oqtane.Modules.Admin.SearchResults @inherits ModuleBase -@inject ISettingService SettingService @implements Oqtane.Interfaces.ISettingsControl +@inject ISettingService SettingService @inject IStringLocalizer Localizer @inject IStringLocalizer SharedLocalizer @@ -15,14 +15,16 @@
@code { - private string resourceType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client"; // for localization + private const string SearchDefaultPageSize = "10"; + + private string resourceType = "Oqtane.Modules.Admin.SearchResults.Settings, Oqtane.Client"; // for localization private string _pageSize; protected override void OnInitialized() { try { - _pageSize = SettingService.GetSetting(ModuleState.Settings, "PageSize", Constants.SearchDefaultPageSize.ToString()); + _pageSize = SettingService.GetSetting(ModuleState.Settings, "PageSize", SearchDefaultPageSize); } catch (Exception ex) { diff --git a/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs deleted file mode 100644 index 3b241ea4..00000000 --- a/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; -using Oqtane.Documentation; -using Oqtane.Models; -using Oqtane.Services; -using Oqtane.Shared; - -namespace Oqtane.Modules.SearchResults.Services -{ - [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] - public class SearchResultsService : ServiceBase, ISearchResultsService, IClientService - { - public SearchResultsService(HttpClient http, SiteState siteState) : base(http, siteState) {} - - private string ApiUrl => CreateApiUrl("SearchResults"); - - public async Task SearchAsync(int moduleId, SearchQuery searchQuery) - { - return await PostJsonAsync(CreateAuthorizationPolicyUrl(ApiUrl, EntityNames.Module, moduleId), searchQuery); - } - } -} diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Index.resx b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Index.resx similarity index 100% rename from Oqtane.Client/Resources/Modules/SearchResults/Index.resx rename to Oqtane.Client/Resources/Modules/Admin/SearchResults/Index.resx diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Settings.resx similarity index 100% rename from Oqtane.Client/Resources/Modules/SearchResults/Settings.resx rename to Oqtane.Client/Resources/Modules/Admin/SearchResults/Settings.resx diff --git a/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs b/Oqtane.Client/Services/Interfaces/ISearchResultsService.cs similarity index 67% rename from Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs rename to Oqtane.Client/Services/Interfaces/ISearchResultsService.cs index 2f1c1308..0671117c 100644 --- a/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs +++ b/Oqtane.Client/Services/Interfaces/ISearchResultsService.cs @@ -3,11 +3,11 @@ using System.Threading.Tasks; using Oqtane.Documentation; using Oqtane.Models; -namespace Oqtane.Modules.SearchResults.Services +namespace Oqtane.Services { [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] public interface ISearchResultsService { - Task SearchAsync(int moduleId, SearchQuery searchQuery); + Task SearchAsync(int moduleId, SearchQuery searchQuery); } } diff --git a/Oqtane.Client/Services/SearchResultsService.cs b/Oqtane.Client/Services/SearchResultsService.cs new file mode 100644 index 00000000..da9687d5 --- /dev/null +++ b/Oqtane.Client/Services/SearchResultsService.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Modules; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + [PrivateApi("Don't show in the documentation, as everything should use the Interface")] + public class SearchResultsService : ServiceBase, ISearchResultsService, IClientService + { + public SearchResultsService(HttpClient http, SiteState siteState) : base(http, siteState) { } + + private string ApiUrl => CreateApiUrl("SearchResults"); + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + return await PostJsonAsync(CreateAuthorizationPolicyUrl(ApiUrl, EntityNames.Module, moduleId), searchQuery); + } + } +} diff --git a/Oqtane.Client/Themes/Controls/Theme/Search.razor b/Oqtane.Client/Themes/Controls/Theme/Search.razor index dbd28b40..fc4980ef 100644 --- a/Oqtane.Client/Themes/Controls/Theme/Search.razor +++ b/Oqtane.Client/Themes/Controls/Theme/Search.razor @@ -16,7 +16,7 @@ @bind-value="_keywords" placeholder="@Localizer["SearchPlaceHolder"]" aria-label="Search" /> - @@ -26,7 +26,7 @@ @code { - private const string SearchResultPagePath = "search-results"; + private const string SearchResultPagePath = "search"; private Page _searchResultsPage; private string _keywords = ""; @@ -48,7 +48,7 @@ var keywords = HttpContext.HttpContext.Request.Form["keywords"]; if (!string.IsNullOrEmpty(keywords) && _searchResultsPage != null) { - var url = NavigateUrl(_searchResultsPage.Path, $"s={keywords}"); + var url = NavigateUrl(_searchResultsPage.Path, $"q={keywords}"); NavigationManager.NavigateTo(url); } } diff --git a/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs b/Oqtane.Server/Controllers/SearchResultsController.cs similarity index 57% rename from Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs rename to Oqtane.Server/Controllers/SearchResultsController.cs index d134b1ae..4b72a464 100644 --- a/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs +++ b/Oqtane.Server/Controllers/SearchResultsController.cs @@ -4,24 +4,22 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Oqtane.Controllers; using Oqtane.Documentation; using Oqtane.Enums; using Oqtane.Infrastructure; -using Oqtane.Modules.SearchResults.Services; +using Oqtane.Services; using Oqtane.Shared; -namespace Oqtane.Modules.SearchResults.Controllers +namespace Oqtane.Controllers { [Route(ControllerRoutes.ApiRoute)] - [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] public class SearchResultsController : ModuleControllerBase { - private readonly ISearchResultsService _searchResultsService; + private readonly ISearchService _searchService; - public SearchResultsController(ISearchResultsService searchResultsService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor) + public SearchResultsController(ISearchService searchService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor) { - _searchResultsService = searchResultsService; + _searchService = searchService; } [HttpPost] @@ -30,9 +28,9 @@ namespace Oqtane.Modules.SearchResults.Controllers { try { - return await _searchResultsService.SearchAsync(AuthEntityId(EntityNames.Module), searchQuery); + return await _searchService.SearchAsync(searchQuery); } - catch(Exception ex) + catch (Exception ex) { _logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery); HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; diff --git a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs index a6612866..d018a4bc 100644 --- a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs +++ b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs @@ -98,6 +98,7 @@ namespace Microsoft.Extensions.DependencyInjection services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -134,7 +135,7 @@ namespace Microsoft.Extensions.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); // managers services.AddTransient(); diff --git a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs index e6b7f216..8ad3e194 100644 --- a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs +++ b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs @@ -11,6 +11,8 @@ namespace Oqtane.Infrastructure { public class SearchIndexJob : HostedServiceBase { + private const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; + public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory) { Name = "Search Index Job"; @@ -54,7 +56,7 @@ namespace Oqtane.Infrastructure private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository) { - var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); if(setting == null) { return null; @@ -65,14 +67,14 @@ namespace Oqtane.Infrastructure private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository) { - var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); if (setting == null) { setting = new Setting { EntityName = EntityNames.Site, EntityId = siteId, - SettingName = Constants.SearchIndexStartTimeSettingName, + SettingName = SearchIndexStartTimeSettingName, SettingValue = Convert.ToString(startTime), }; diff --git a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs index 99bb8aaa..447678ed 100644 --- a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs +++ b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs @@ -138,7 +138,7 @@ namespace Oqtane.SiteTemplates Name = "Search Results", Parent = "", Order = 7, - Path = "search-results", + Path = "search", Icon = "oi oi-magnifying-glass", IsNavigation = false, IsPersonalizable = false, @@ -148,7 +148,7 @@ namespace Oqtane.SiteTemplates new Permission(PermissionNames.Edit, RoleNames.Admin, true) }, PageTemplateModules = new List { - new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, PermissionList = new List { new Permission(PermissionNames.View, RoleNames.Everyone, true), new Permission(PermissionNames.View, RoleNames.Admin, true), diff --git a/Oqtane.Server/Infrastructure/UpgradeManager.cs b/Oqtane.Server/Infrastructure/UpgradeManager.cs index 3f0dd0f6..8737dd65 100644 --- a/Oqtane.Server/Infrastructure/UpgradeManager.cs +++ b/Oqtane.Server/Infrastructure/UpgradeManager.cs @@ -66,6 +66,9 @@ namespace Oqtane.Infrastructure case "5.1.0": Upgrade_5_1_0(tenant, scope); break; + case "5.2.0": + Upgrade_5_2_0(tenant, scope); + break; } } } @@ -385,5 +388,48 @@ namespace Oqtane.Infrastructure } } + + private void Upgrade_5_2_0(Tenant tenant, IServiceScope scope) + { + CreateSearchResultsPages(tenant, scope); + } + + private void CreateSearchResultsPages(Tenant tenant, IServiceScope scope) + { + var pageTemplates = new List(); + pageTemplates.Add(new PageTemplate + { + Name = "Search Results", + Parent = "", + Path = "search", + Icon = "oi oi-magnifying-glass", + IsNavigation = false, + IsPersonalizable = false, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + }, + PageTemplateModules = new List { + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + } + } + } + }); + + var pages = scope.ServiceProvider.GetRequiredService(); + var sites = scope.ServiceProvider.GetRequiredService(); + foreach (var site in sites.GetSites().ToList()) + { + if (!pages.GetPages(site.SiteId).ToList().Where(item => item.Path == "search").Any()) + { + sites.CreatePages(site, pageTemplates, null); + } + } + } } } diff --git a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs index fec739d2..b68c993f 100644 --- a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs +++ b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs @@ -12,6 +12,8 @@ namespace Oqtane.Managers.Search { public class ModuleSearchIndexManager : SearchIndexManagerBase { + public const int ModuleSearchIndexManagerPriority = 200; + private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly IPageModuleRepository _pageModuleRepostory; @@ -30,35 +32,41 @@ namespace Oqtane.Managers.Search _pageRepository = pageRepository; } - public override string Name => Constants.ModuleSearchIndexManagerName; + public override string Name => EntityNames.Module; - public override int Priority => Constants.ModuleSearchIndexManagerPriority; + public override int Priority => ModuleSearchIndexManagerPriority; - public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) { var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId); - var searchDocuments = new List(); + var searchContentList = new List(); foreach(var pageModule in pageModules) { + var page = _pageRepository.GetPage(pageModule.PageId); + if(page == null || SearchUtils.IsSystemPage(page)) + { + continue; + } + var module = pageModule.Module; if (module.ModuleDefinition.ServerManagerType != "") { _logger.LogDebug($"Search: Begin index module {module.ModuleId}."); var type = Type.GetType(module.ModuleDefinition.ServerManagerType); - if (type?.GetInterface("IModuleSearch") != null) + if (type?.GetInterface(nameof(ISearchable)) != null) { try { - var moduleSearch = (IModuleSearch)ActivatorUtilities.CreateInstance(_serviceProvider, type); - var documents = moduleSearch.GetSearchDocuments(module, startTime.GetValueOrDefault(DateTime.MinValue)); - if(documents != null) + var moduleSearch = (ISearchable)ActivatorUtilities.CreateInstance(_serviceProvider, type); + var contentList = moduleSearch.GetSearchContentList(module, startTime.GetValueOrDefault(DateTime.MinValue)); + if(contentList != null) { - foreach(var document in documents) + foreach(var searchContent in contentList) { - SaveModuleMetaData(document, pageModule); + SaveModuleMetaData(searchContent, pageModule); - searchDocuments.Add(document); + searchContentList.Add(searchContent); } } @@ -73,54 +81,65 @@ namespace Oqtane.Managers.Search } } - processSearchDocuments(searchDocuments); + processSearchContent(searchContentList); - return searchDocuments.Count; + return searchContentList.Count; } - private void SaveModuleMetaData(SearchDocument document, PageModule pageModule) + private void SaveModuleMetaData(SearchContent searchContent, PageModule pageModule) { - - document.EntryId = pageModule.ModuleId; - document.IndexerName = Name; - document.SiteId = pageModule.Module.SiteId; - document.LanguageCode = string.Empty; + searchContent.SiteId = pageModule.Module.SiteId; - if(document.ModifiedTime == DateTime.MinValue) + if(string.IsNullOrEmpty(searchContent.EntityName)) { - document.ModifiedTime = pageModule.ModifiedOn; + searchContent.EntityName = EntityNames.Module; } - if (string.IsNullOrEmpty(document.AdditionalContent)) + if(searchContent.EntityId == 0) { - document.AdditionalContent = string.Empty; + searchContent.EntityId = pageModule.ModuleId; + } + + if (searchContent.IsActive) + { + searchContent.IsActive = !pageModule.Module.IsDeleted; + } + + if (searchContent.ModifiedTime == DateTime.MinValue) + { + searchContent.ModifiedTime = pageModule.ModifiedOn; + } + + if (string.IsNullOrEmpty(searchContent.AdditionalContent)) + { + searchContent.AdditionalContent = string.Empty; } var page = _pageRepository.GetPage(pageModule.PageId); - if (string.IsNullOrEmpty(document.Url) && page != null) + if (string.IsNullOrEmpty(searchContent.Url) && page != null) { - document.Url = page.Url; + searchContent.Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; } - if (string.IsNullOrEmpty(document.Title) && page != null) + if (string.IsNullOrEmpty(searchContent.Title) && page != null) { - document.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name; + searchContent.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name; } - if (document.Properties == null) + if (searchContent.Properties == null) { - document.Properties = new List(); + searchContent.Properties = new List(); } - if(!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + if(!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) { - document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); } - if (!document.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) + if (!searchContent.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) { - document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); } } } diff --git a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs index 55789c23..c40f5f0b 100644 --- a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs +++ b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs @@ -11,7 +11,7 @@ namespace Oqtane.Managers.Search { public class ModuleSearchResultManager : ISearchResultManager { - public string Name => Constants.ModuleSearchIndexManagerName; + public string Name => EntityNames.Module; private readonly IServiceProvider _serviceProvider; @@ -37,26 +37,17 @@ namespace Oqtane.Managers.Search return string.Empty; } - public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) + public bool Visible(SearchContent searchResult, SearchQuery searchQuery) { var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; - var moduleIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchModuleIdPropertyName)?.Value ?? string.Empty; - if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId) - && !string.IsNullOrEmpty(moduleIdValue) && int.TryParse(moduleIdValue, out int moduleId)) + if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)) { - return CanViewPage(pageId, searchQuery.User) && CanViewModule(moduleId, searchQuery.User); + return CanViewPage(pageId, searchQuery.User); } return false; } - private bool CanViewModule(int moduleId, User user) - { - var moduleRepository = _serviceProvider.GetRequiredService(); - var module = moduleRepository.GetModule(moduleId); - return module != null && !module.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, module.PermissionList); - } - private bool CanViewPage(int pageId, User user) { var pageRepository = _serviceProvider.GetRequiredService(); diff --git a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs index 68c4af02..49599804 100644 --- a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs +++ b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs @@ -14,6 +14,8 @@ namespace Oqtane.Managers.Search { public class PageSearchIndexManager : SearchIndexManagerBase { + private const int PageSearchIndexManagerPriority = 100; + private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly IPageRepository _pageRepository; @@ -29,50 +31,50 @@ namespace Oqtane.Managers.Search _pageRepository = pageRepository; } - public override string Name => Constants.PageSearchIndexManagerName; + public override string Name => EntityNames.Page; - public override int Priority => Constants.PageSearchIndexManagerPriority; + public override int Priority => PageSearchIndexManagerPriority; - public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) { var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue); var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue); - var searchDocuments = new List(); + var searchContentList = new List(); foreach(var page in pages) { try { - if(IsSystemPage(page)) + if(SearchUtils.IsSystemPage(page)) { continue; } - var document = new SearchDocument + var searchContent = new SearchContent { - EntryId = page.PageId, - IndexerName = Name, + EntityName = EntityNames.Page, + EntityId = page.PageId, SiteId = page.SiteId, - LanguageCode = string.Empty, ModifiedTime = page.ModifiedOn, AdditionalContent = string.Empty, - Url = page.Url ?? string.Empty, + Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}", Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, Description = string.Empty, - Body = $"{page.Name} {page.Title}" + Body = $"{page.Name} {page.Title}", + IsActive = !page.IsDeleted && Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) }; - if (document.Properties == null) + if (searchContent.Properties == null) { - document.Properties = new List(); + searchContent.Properties = new List(); } - if (!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + if (!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) { - document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); } - searchDocuments.Add(document); + searchContentList.Add(searchContent); } catch(Exception ex) { @@ -81,14 +83,9 @@ namespace Oqtane.Managers.Search } } - processSearchDocuments(searchDocuments); + processSearchContent(searchContentList); - return searchDocuments.Count; - } - - private bool IsSystemPage(Models.Page page) - { - return page.Path.Contains("admin") || page.Path == "login" || page.Path == "register" || page.Path == "profile"; + return searchContentList.Count; } } } diff --git a/Oqtane.Server/Managers/Search/PageSearchResultManager.cs b/Oqtane.Server/Managers/Search/PageSearchResultManager.cs deleted file mode 100644 index fd5456a4..00000000 --- a/Oqtane.Server/Managers/Search/PageSearchResultManager.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Oqtane.Models; -using Oqtane.Repository; -using Oqtane.Security; -using Oqtane.Services; -using Oqtane.Shared; - -namespace Oqtane.Managers.Search -{ - public class PageSearchResultManager : ISearchResultManager - { - public string Name => Constants.PageSearchIndexManagerName; - - private readonly IServiceProvider _serviceProvider; - - public PageSearchResultManager( - IServiceProvider serviceProvider) - { - _serviceProvider = serviceProvider; - } - - public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) - { - var pageRepository = _serviceProvider.GetRequiredService(); - var page = pageRepository.GetPage(searchResult.EntryId); - if (page != null) - { - return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; - } - - return string.Empty; - } - - public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) - { - var pageRepository = _serviceProvider.GetRequiredService(); - var page = pageRepository.GetPage(searchResult.EntryId); - - return page != null && !page.IsDeleted - && UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.View, page.PermissionList) - && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.Edit, page.PermissionList)); - } - } -} diff --git a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs index 0ecb2880..779a950d 100644 --- a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs +++ b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs @@ -10,6 +10,8 @@ namespace Oqtane.Managers.Search { public abstract class SearchIndexManagerBase : ISearchIndexManager { + private const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; + private readonly IServiceProvider _serviceProvider; public SearchIndexManagerBase(IServiceProvider serviceProvider) @@ -21,11 +23,11 @@ namespace Oqtane.Managers.Search public abstract string Name { get; } - public abstract int IndexDocuments(int siteId, DateTime? startDate, Action> processSearchDocuments, Action handleError); + public abstract int IndexContent(int siteId, DateTime? startDate, Action> processSearchContent, Action handleError); public virtual bool IsIndexEnabled(int siteId) { - var settingName = string.Format(Constants.SearchIndexManagerEnabledSettingFormat, Name); + var settingName = string.Format(SearchIndexManagerEnabledSettingFormat, Name); var settingRepository = _serviceProvider.GetRequiredService(); var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName); return setting == null || setting.SettingValue == "true"; diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs similarity index 58% rename from Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs rename to Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs index 2f5b6d8d..4cf2aa4a 100644 --- a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs @@ -2,25 +2,26 @@ using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Migrations.Operations; using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; using Oqtane.Databases.Interfaces; +using Oqtane.Models; namespace Oqtane.Migrations.EntityBuilders { - public class SearchDocumentEntityBuilder : AuditableBaseEntityBuilder + public class SearchContentEntityBuilder : AuditableBaseEntityBuilder { - private const string _entityTableName = "SearchDocument"; - private readonly PrimaryKey _primaryKey = new("PK_SearchDocument", x => x.SearchDocumentId); + private const string _entityTableName = "SearchContent"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContent", x => x.SearchContentId); - public SearchDocumentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + public SearchContentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) { EntityTableName = _entityTableName; PrimaryKey = _primaryKey; } - protected override SearchDocumentEntityBuilder BuildTable(ColumnsBuilder table) + protected override SearchContentEntityBuilder BuildTable(ColumnsBuilder table) { - SearchDocumentId = AddAutoIncrementColumn(table, "SearchDocumentId"); - EntryId = AddIntegerColumn(table, "EntryId"); - IndexerName = AddStringColumn(table, "IndexerName", 50); + SearchContentId = AddAutoIncrementColumn(table, "SearchContentId"); + EntityName = AddStringColumn(table, "EntityName", 50); + EntityId = AddIntegerColumn(table, "EntityId"); SiteId = AddIntegerColumn(table, "SiteId"); Title = AddStringColumn(table, "Title", 255); Description = AddMaxStringColumn(table, "Description"); @@ -29,18 +30,17 @@ namespace Oqtane.Migrations.EntityBuilders ModifiedTime = AddDateTimeColumn(table, "ModifiedTime"); IsActive = AddBooleanColumn(table, "IsActive"); AdditionalContent = AddMaxStringColumn(table, "AdditionalContent"); - LanguageCode = AddStringColumn(table, "LanguageCode", 20); AddAuditableColumns(table); return this; } - public OperationBuilder SearchDocumentId { get; private set; } + public OperationBuilder SearchContentId { get; private set; } - public OperationBuilder EntryId { get; private set; } + public OperationBuilder EntityName { get; private set; } - public OperationBuilder IndexerName { get; private set; } + public OperationBuilder EntityId { get; private set; } public OperationBuilder SiteId { get; private set; } @@ -57,8 +57,5 @@ namespace Oqtane.Migrations.EntityBuilders public OperationBuilder IsActive { get; private set; } public OperationBuilder AdditionalContent { get; private set; } - - public OperationBuilder LanguageCode { get; private set; } - } } diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs new file mode 100644 index 00000000..01870cd1 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentPropertyEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentProperty"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentProperty", x => x.PropertyId); + private readonly ForeignKey _searchContentForeignKey = new("FK_SearchContentProperty_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade); + + public SearchContentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchContentForeignKey); + } + + protected override SearchContentPropertyEntityBuilder BuildTable(ColumnsBuilder table) + { + PropertyId = AddAutoIncrementColumn(table, "PropertyId"); + SearchContentId = AddIntegerColumn(table, "SearchContentId"); + Name = AddStringColumn(table, "Name", 50); + Value = AddStringColumn(table, "Value", 50); + + return this; + } + + public OperationBuilder PropertyId { get; private set; } + + public OperationBuilder SearchContentId { get; private set; } + + public OperationBuilder Name { get; private set; } + + public OperationBuilder Value { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs new file mode 100644 index 00000000..2eb845fc --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs @@ -0,0 +1,42 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentWordsEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentWords"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentWords", x => x.WordId); + private readonly ForeignKey _searchContentForeignKey = new("FK_SearchContentWords_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade); + private readonly ForeignKey _wordSourceForeignKey = new("FK_SearchContentWords_WordSource", x => x.WordSourceId, "SearchContentWordSource", "WordSourceId", ReferentialAction.Cascade); + + public SearchContentWordsEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchContentForeignKey); + ForeignKeys.Add(_wordSourceForeignKey); + } + + protected override SearchContentWordsEntityBuilder BuildTable(ColumnsBuilder table) + { + WordId = AddAutoIncrementColumn(table, "WordId"); + SearchContentId = AddIntegerColumn(table, "SearchContentId"); + WordSourceId = AddIntegerColumn(table, "WordSourceId"); + Count = AddIntegerColumn(table, "Count"); + + return this; + } + + public OperationBuilder WordId { get; private set; } + + public OperationBuilder SearchContentId { get; private set; } + + public OperationBuilder WordSourceId { get; private set; } + + public OperationBuilder Count { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs new file mode 100644 index 00000000..0787a3a4 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentWordSourceEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentWordSource"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentWordSource", x => x.WordSourceId); + + public SearchContentWordSourceEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + } + + protected override SearchContentWordSourceEntityBuilder BuildTable(ColumnsBuilder table) + { + WordSourceId = AddAutoIncrementColumn(table, "WordSourceId"); + Word = AddStringColumn(table, "Word", 255); + + return this; + } + + public OperationBuilder WordSourceId { get; private set; } + + public OperationBuilder Word { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs deleted file mode 100644 index a591aec2..00000000 --- a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Migrations.Operations; -using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; -using Oqtane.Databases.Interfaces; - -namespace Oqtane.Migrations.EntityBuilders -{ - public class SearchDocumentPropertyEntityBuilder : BaseEntityBuilder - { - private const string _entityTableName = "SearchDocumentProperty"; - private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentProperty", x => x.PropertyId); - private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentProperty_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); - - public SearchDocumentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) - { - EntityTableName = _entityTableName; - PrimaryKey = _primaryKey; - - ForeignKeys.Add(_searchDocumentForeignKey); - } - - protected override SearchDocumentPropertyEntityBuilder BuildTable(ColumnsBuilder table) - { - PropertyId = AddAutoIncrementColumn(table, "PropertyId"); - SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); - Name = AddStringColumn(table, "Name", 50); - Value = AddStringColumn(table, "Value", 50); - - return this; - } - - public OperationBuilder PropertyId { get; private set; } - - public OperationBuilder SearchDocumentId { get; private set; } - - public OperationBuilder Name { get; private set; } - - public OperationBuilder Value { get; private set; } - } -} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs deleted file mode 100644 index bd864f22..00000000 --- a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Migrations.Operations; -using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; -using Oqtane.Databases.Interfaces; - -namespace Oqtane.Migrations.EntityBuilders -{ - public class SearchDocumentTagEntityBuilder : BaseEntityBuilder - { - private const string _entityTableName = "SearchDocumentTag"; - private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentTag", x => x.TagId); - private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentTag_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); - - public SearchDocumentTagEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) - { - EntityTableName = _entityTableName; - PrimaryKey = _primaryKey; - - ForeignKeys.Add(_searchDocumentForeignKey); - } - - protected override SearchDocumentTagEntityBuilder BuildTable(ColumnsBuilder table) - { - TagId = AddAutoIncrementColumn(table, "TagId"); - SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); - Tag = AddStringColumn(table, "Tag", 50); - - return this; - } - - public OperationBuilder TagId { get; private set; } - - public OperationBuilder SearchDocumentId { get; private set; } - - public OperationBuilder Tag { get; private set; } - } -} diff --git a/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs index f32a8137..c66a4fe5 100644 --- a/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs +++ b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs @@ -17,26 +17,34 @@ namespace Oqtane.Migrations.Tenant protected override void Up(MigrationBuilder migrationBuilder) { - var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentEntityBuilder.Create(); + var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentEntityBuilder.Create(); - var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentPropertyEntityBuilder.Create(); + var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentPropertyEntityBuilder.Create(); - var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentTagEntityBuilder.Create(); + var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordSourceEntityBuilder.Create(); + searchContentWordSourceEntityBuilder.AddIndex("IX_SearchContentWordSource", "Word", true); + + var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordsEntityBuilder.Create(); } protected override void Down(MigrationBuilder migrationBuilder) { - var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentPropertyEntityBuilder.Drop(); + var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordsEntityBuilder.Drop(); - var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentTagEntityBuilder.Drop(); + var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordSourceEntityBuilder.DropIndex("IX_SearchContentWordSource"); + searchContentWordSourceEntityBuilder.Drop(); - var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); - searchDocumentEntityBuilder.Drop(); + var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentPropertyEntityBuilder.Drop(); + + var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentEntityBuilder.Drop(); } } } diff --git a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs index 8cabf391..e5cfb8de 100644 --- a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs +++ b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs @@ -17,7 +17,7 @@ using System; namespace Oqtane.Modules.HtmlText.Manager { [PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")] - public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, IModuleSearch + public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, ISearchable { private readonly IServiceProvider _serviceProvider; private readonly IHtmlTextRepository _htmlText; @@ -48,25 +48,25 @@ namespace Oqtane.Modules.HtmlText.Manager return content; } - public IList GetSearchDocuments(Module module, DateTime startDate) + public IList GetSearchContentList(Module module, DateTime startDate) { - var searchDocuments = new List(); + var searchContentList = new List(); var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId); if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate)) { var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First(); - searchDocuments.Add(new SearchDocument + searchContentList.Add(new SearchContent { Title = module.Title, Description = string.Empty, - Body = SearchUtils.Clean(htmltext.Content, true), + Body = htmltext.Content, ModifiedTime = htmltext.ModifiedOn }); } - return searchDocuments; + return searchContentList; } public void ImportModule(Module module, string content, string version) diff --git a/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs deleted file mode 100644 index 3823bac6..00000000 --- a/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Oqtane.Documentation; -using Oqtane.Infrastructure; -using Oqtane.Models; -using Oqtane.Services; - -namespace Oqtane.Modules.SearchResults.Services -{ - [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] - public class ServerSearchResultsService : ISearchResultsService, ITransientService - { - private readonly ILogManager _logger; - private readonly IHttpContextAccessor _accessor; - private readonly Alias _alias; - private readonly ISearchService _searchService; - - public ServerSearchResultsService( - ITenantManager tenantManager, - ILogManager logger, - IHttpContextAccessor accessor, - ISearchService searchService) - { - _logger = logger; - _accessor = accessor; - _alias = tenantManager.GetAlias(); - _searchService = searchService; - } - - public async Task SearchAsync(int moduleId, SearchQuery searchQuery) - { - var results = await _searchService.SearchAsync(searchQuery); - return results; - } - } -} diff --git a/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs b/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs deleted file mode 100644 index 25d2d15f..00000000 --- a/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.DependencyInjection; -using Oqtane.Infrastructure; -using Oqtane.Modules.SearchResults.Services; - -namespace Oqtane.Modules.SearchResults.Startup -{ - public class ServerStartup : IServerStartup - { - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) - { - } - - public void ConfigureMvc(IMvcBuilder mvcBuilder) - { - } - - public void ConfigureServices(IServiceCollection services) - { - services.AddTransient(); - } - } -} diff --git a/Oqtane.Server/Oqtane.Server.csproj b/Oqtane.Server/Oqtane.Server.csproj index f661e715..52776903 100644 --- a/Oqtane.Server/Oqtane.Server.csproj +++ b/Oqtane.Server/Oqtane.Server.csproj @@ -33,6 +33,7 @@ + diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs index ac2ed576..74884934 100644 --- a/Oqtane.Server/Providers/DatabaseSearchProvider.cs +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Xml; +using HtmlAgilityPack; using Oqtane.Models; using Oqtane.Repository; using Oqtane.Services; @@ -13,27 +16,28 @@ namespace Oqtane.Providers { public class DatabaseSearchProvider : ISearchProvider { - private readonly ISearchDocumentRepository _searchDocumentRepository; + private readonly ISearchContentRepository _searchContentRepository; private const float TitleBoost = 100f; private const float DescriptionBoost = 10f; private const float BodyBoost = 10f; private const float AdditionalContentBoost = 5f; - + private const string IgnoreWords = "the,be,to,of,and,a,i,in,that,have,it,for,not,on,with,he,as,you,do,at,this,but,his,by,from,they,we,say,her,she,or,an,will,my,one,all,would,there,their,what,so,up,out,if,about,who,get,which,go,me,when,make,can,like,time,no,just,him,know,take,people,into,year,your,good,some,could,them,see,other,than,then,now,look,only,come,its,over,think,also,back,after,use,two,how,our,work,first,well,way,even,new,want,because,any,these,give,day,most,us"; + private const int WordMinLength = 3; public string Name => Constants.DefaultSearchProviderName; - public DatabaseSearchProvider(ISearchDocumentRepository searchDocumentRepository) + public DatabaseSearchProvider(ISearchContentRepository searchContentRepository) { - _searchDocumentRepository = searchDocumentRepository; + _searchContentRepository = searchContentRepository; } public void Commit() { } - public void DeleteDocument(string id) + public void DeleteSearchContent(string id) { - _searchDocumentRepository.DeleteSearchDocument(id); + _searchContentRepository.DeleteSearchContent(id); } public bool Optimize() @@ -43,25 +47,28 @@ namespace Oqtane.Providers public void ResetIndex() { - _searchDocumentRepository.DeleteAllSearchDocuments(); + _searchContentRepository.DeleteAllSearchContent(); } - public void SaveDocument(SearchDocument document, bool autoCommit = false) + public void SaveSearchContent(SearchContent searchContent, bool autoCommit = false) { //remove exist document - _searchDocumentRepository.DeleteSearchDocument(document.IndexerName, document.EntryId); + _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId); - _searchDocumentRepository.AddSearchDocument(document); + _searchContentRepository.AddSearchContent(searchContent); + + //save the index words + AnalyzeSearchContent(searchContent); } - public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) + public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) { var totalResults = 0; - var documents = await _searchDocumentRepository.GetSearchDocumentsAsync(searchQuery); + var searchContentList = await _searchContentRepository.GetSearchContentListAsync(searchQuery); - //convert the search documents to search results. - var results = documents + //convert the search content to search results. + var results = searchContentList .Where(i => validateFunc(i, searchQuery)) .Select(i => ConvertToSearchResult(i, searchQuery)); @@ -99,7 +106,7 @@ namespace Oqtane.Providers //remove duplicated results based on page id for Page and Module types results = results.DistinctBy(i => { - if (i.IndexerName == Constants.PageSearchIndexManagerName || i.IndexerName == Constants.ModuleSearchIndexManagerName) + if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module) { var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey; @@ -119,45 +126,44 @@ namespace Oqtane.Providers }; } - private SearchResult ConvertToSearchResult(SearchDocument searchDocument, SearchQuery searchQuery) + private SearchResult ConvertToSearchResult(SearchContent searchContent, SearchQuery searchQuery) { var searchResult = new SearchResult() { - SearchDocumentId = searchDocument.SearchDocumentId, - SiteId = searchDocument.SiteId, - IndexerName = searchDocument.IndexerName, - EntryId = searchDocument.EntryId, - Title = searchDocument.Title, - Description = searchDocument.Description, - Body = searchDocument.Body, - Url = searchDocument.Url, - ModifiedTime = searchDocument.ModifiedTime, - Tags = searchDocument.Tags, - Properties = searchDocument.Properties, - Snippet = BuildSnippet(searchDocument, searchQuery), - Score = CalculateScore(searchDocument, searchQuery) + SearchContentId = searchContent.SearchContentId, + SiteId = searchContent.SiteId, + EntityName = searchContent.EntityName, + EntityId = searchContent.EntityId, + Title = searchContent.Title, + Description = searchContent.Description, + Body = searchContent.Body, + Url = searchContent.Url, + ModifiedTime = searchContent.ModifiedTime, + Properties = searchContent.Properties, + Snippet = BuildSnippet(searchContent, searchQuery), + Score = CalculateScore(searchContent, searchQuery) }; return searchResult; } - private float CalculateScore(SearchDocument searchDocument, SearchQuery searchQuery) + private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery) { var score = 0f; foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) { - score += Regex.Matches(searchDocument.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost; - score += Regex.Matches(searchDocument.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost; - score += Regex.Matches(searchDocument.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost; - score += Regex.Matches(searchDocument.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost; + score += Regex.Matches(searchContent.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost; + score += Regex.Matches(searchContent.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost; + score += Regex.Matches(searchContent.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost; + score += Regex.Matches(searchContent.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost; } return score / 100; } - private string BuildSnippet(SearchDocument searchDocument, SearchQuery searchQuery) + private string BuildSnippet(SearchContent searchContent, SearchQuery searchQuery) { - var content = $"{searchDocument.Title} {searchDocument.Description} {searchDocument.Body}"; + var content = $"{searchContent.Title} {searchContent.Description} {searchContent.Body}"; var snippet = string.Empty; foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) { @@ -198,5 +204,92 @@ namespace Oqtane.Providers return snippet; } + + private void AnalyzeSearchContent(SearchContent searchContent) + { + //analyze the search content and save the index words + var indexContent = $"{searchContent.Title} {searchContent.Description} {searchContent.Body} {searchContent.AdditionalContent}"; + var words = GetWords(indexContent, WordMinLength); + var existWords = _searchContentRepository.GetWords(searchContent.SearchContentId); + foreach (var kvp in words) + { + var word = existWords.FirstOrDefault(i => i.WordSource.Word == kvp.Key); + if (word != null) + { + word.Count = kvp.Value; + _searchContentRepository.UpdateSearchContentWords(word); + } + else + { + var wordSource = _searchContentRepository.GetSearchContentWordSource(kvp.Key); + if (wordSource == null) + { + wordSource = _searchContentRepository.AddSearchContentWordSource(new SearchContentWordSource { Word = kvp.Key }); + } + + word = new SearchContentWords + { + SearchContentId = searchContent.SearchContentId, + WordSourceId = wordSource.WordSourceId, + Count = kvp.Value + }; + + _searchContentRepository.AddSearchContentWords(word); + } + } + } + + private static Dictionary GetWords(string content, int minLength) + { + content = WebUtility.HtmlDecode(content); + + var words = new Dictionary(); + var ignoreWords = IgnoreWords.Split(','); + + var page = new HtmlDocument(); + page.LoadHtml(content); + + var phrases = page.DocumentNode.Descendants().Where(i => + i.NodeType == HtmlNodeType.Text && + i.ParentNode.Name != "script" && + i.ParentNode.Name != "style" && + !string.IsNullOrEmpty(i.InnerText.Trim()) + ).Select(i => FormatText(i.InnerText)); + + foreach (var phrase in phrases) + { + if (!string.IsNullOrEmpty(phrase)) + { + foreach (var word in phrase.Split(' ')) + { + if (word.Length >= minLength && !ignoreWords.Contains(word)) + { + if (!words.ContainsKey(word)) + { + words.Add(word, 1); + } + else + { + words[word] += 1; + } + } + } + } + } + + return words; + } + + private static string FormatText(string text) + { + text = HtmlEntity.DeEntitize(text); + foreach (var punctuation in ".?!,;:-_()[]{}'\"/\\".ToCharArray()) + { + text = text.Replace(punctuation, ' '); + } + text = text.Replace(" ", " ").ToLower().Trim(); + return text; + + } } } diff --git a/Oqtane.Server/Repository/Context/TenantDBContext.cs b/Oqtane.Server/Repository/Context/TenantDBContext.cs index cbe57631..cb0dcb28 100644 --- a/Oqtane.Server/Repository/Context/TenantDBContext.cs +++ b/Oqtane.Server/Repository/Context/TenantDBContext.cs @@ -29,8 +29,9 @@ namespace Oqtane.Repository public virtual DbSet Language { get; set; } public virtual DbSet Visitor { get; set; } public virtual DbSet UrlMapping { get; set; } - public virtual DbSet SearchDocument { get; set; } - public virtual DbSet SearchDocumentProperty { get; set; } - public virtual DbSet SearchDocumentTag { get; set; } + public virtual DbSet SearchContent { get; set; } + public virtual DbSet SearchContentProperty { get; set; } + public virtual DbSet SearchContentWords { get; set; } + public virtual DbSet SearchContentWordSource { get; set; } } } diff --git a/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs b/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs new file mode 100644 index 00000000..868265b0 --- /dev/null +++ b/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Repository +{ + public interface ISearchContentRepository + { + Task> GetSearchContentListAsync(SearchQuery searchQuery); + SearchContent AddSearchContent(SearchContent searchContent); + void DeleteSearchContent(int searchContentId); + void DeleteSearchContent(string entityName, int entryId); + void DeleteSearchContent(string uniqueKey); + void DeleteAllSearchContent(); + + SearchContentWordSource GetSearchContentWordSource(string word); + SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource); + + IEnumerable GetWords(int searchContentId); + SearchContentWords AddSearchContentWords(SearchContentWords word); + SearchContentWords UpdateSearchContentWords(SearchContentWords word); + } +} diff --git a/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs b/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs deleted file mode 100644 index 6128c40c..00000000 --- a/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Oqtane.Models; - -namespace Oqtane.Repository -{ - public interface ISearchDocumentRepository - { - Task> GetSearchDocumentsAsync(SearchQuery searchQuery); - SearchDocument AddSearchDocument(SearchDocument searchDocument); - void DeleteSearchDocument(int searchDocumentId); - void DeleteSearchDocument(string indexerName, int entryId); - void DeleteSearchDocument(string uniqueKey); - void DeleteAllSearchDocuments(); - } -} diff --git a/Oqtane.Server/Repository/SearchContentRepository.cs b/Oqtane.Server/Repository/SearchContentRepository.cs new file mode 100644 index 00000000..c1577d81 --- /dev/null +++ b/Oqtane.Server/Repository/SearchContentRepository.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Repository +{ + public class SearchContentRepository : ISearchContentRepository + { + private readonly IDbContextFactory _dbContextFactory; + + public SearchContentRepository(IDbContextFactory dbContextFactory) + { + _dbContextFactory = dbContextFactory; + } + + public async Task> GetSearchContentListAsync(SearchQuery searchQuery) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContentList = db.SearchContent.AsNoTracking() + .Include(i => i.Properties) + .Where(i => i.SiteId == searchQuery.SiteId && i.IsActive); + + if (searchQuery.EntityNames != null && searchQuery.EntityNames.Any()) + { + searchContentList = searchContentList.Where(i => searchQuery.EntityNames.Contains(i.EntityName)); + } + + if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue) + { + searchContentList = searchContentList.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc); + } + + if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue) + { + searchContentList = searchContentList.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc); + } + + if (searchQuery.Properties != null && searchQuery.Properties.Any()) + { + foreach (var property in searchQuery.Properties) + { + searchContentList = searchContentList.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value)); + } + } + + var filteredContentList = new List(); + if (!string.IsNullOrEmpty(searchQuery.Keywords)) + { + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + filteredContentList.AddRange(await searchContentList.Where(i => i.Words.Any(w => w.WordSource.Word.StartsWith(keyword))).ToListAsync()); + } + } + + return filteredContentList.DistinctBy(i => i.UniqueKey); + } + + public SearchContent AddSearchContent(SearchContent searchContent) + { + using var context = _dbContextFactory.CreateDbContext(); + context.SearchContent.Add(searchContent); + + if(searchContent.Properties != null && searchContent.Properties.Any()) + { + foreach(var property in searchContent.Properties) + { + property.SearchContentId = searchContent.SearchContentId; + context.SearchContentProperty.Add(property); + } + } + + context.SaveChanges(); + + return searchContent; + } + + public void DeleteSearchContent(int searchContentId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.Find(searchContentId); + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + + public void DeleteSearchContent(string entityName, int entryId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.FirstOrDefault(i => i.EntityName == entityName && i.EntityId == entryId); + if(searchContent != null) + { + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + } + + public void DeleteSearchContent(string uniqueKey) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.FirstOrDefault(i => (i.EntityName + ":" + i.EntityId) == uniqueKey); + if (searchContent != null) + { + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + } + + public void DeleteAllSearchContent() + { + using var db = _dbContextFactory.CreateDbContext(); + db.SearchContent.RemoveRange(db.SearchContent); + db.SaveChanges(); + } + + public SearchContentWordSource GetSearchContentWordSource(string word) + { + if(string.IsNullOrEmpty(word)) + { + return null; + } + + using var db = _dbContextFactory.CreateDbContext(); + return db.SearchContentWordSource.FirstOrDefault(i => i.Word == word); + } + + public SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.SearchContentWordSource.Add(wordSource); + db.SaveChanges(); + + return wordSource; + } + + public IEnumerable GetWords(int searchContentId) + { + using var db = _dbContextFactory.CreateDbContext(); + return db.SearchContentWords + .Include(i => i.WordSource) + .Where(i => i.SearchContentId == searchContentId).ToList(); + } + + public SearchContentWords AddSearchContentWords(SearchContentWords word) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.SearchContentWords.Add(word); + db.SaveChanges(); + + return word; + } + + public SearchContentWords UpdateSearchContentWords(SearchContentWords word) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.Entry(word).State = EntityState.Modified; + db.SaveChanges(); + + return word; + } + } +} diff --git a/Oqtane.Server/Repository/SearchDocumentRepository.cs b/Oqtane.Server/Repository/SearchDocumentRepository.cs deleted file mode 100644 index bdceb52e..00000000 --- a/Oqtane.Server/Repository/SearchDocumentRepository.cs +++ /dev/null @@ -1,136 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; -using Oqtane.Models; -using Oqtane.Shared; - -namespace Oqtane.Repository -{ - public class SearchDocumentRepository : ISearchDocumentRepository - { - private readonly IDbContextFactory _dbContextFactory; - - public SearchDocumentRepository(IDbContextFactory dbContextFactory) - { - _dbContextFactory = dbContextFactory; - } - - public async Task> GetSearchDocumentsAsync(SearchQuery searchQuery) - { - using var db = _dbContextFactory.CreateDbContext(); - var documents = db.SearchDocument.AsNoTracking() - .Include(i => i.Properties) - .Include(i => i.Tags) - .Where(i => i.SiteId == searchQuery.SiteId); - - if (searchQuery.Sources != null && searchQuery.Sources.Any()) - { - documents = documents.Where(i => searchQuery.Sources.Contains(i.IndexerName)); - } - - if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue) - { - documents = documents.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc); - } - - if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue) - { - documents = documents.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc); - } - - if (searchQuery.Tags != null && searchQuery.Tags.Any()) - { - foreach (var tag in searchQuery.Tags) - { - documents = documents.Where(i => i.Tags.Any(t => t.Tag == tag)); - } - } - - if (searchQuery.Properties != null && searchQuery.Properties.Any()) - { - foreach (var property in searchQuery.Properties) - { - documents = documents.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value)); - } - } - - var filteredDocuments = new List(); - if (!string.IsNullOrEmpty(searchQuery.Keywords)) - { - foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) - { - filteredDocuments.AddRange(await documents.Where(i => i.Title.Contains(keyword) || i.Description.Contains(keyword) || i.Body.Contains(keyword)).ToListAsync()); - } - } - - return filteredDocuments.DistinctBy(i => i.UniqueKey); - } - - public SearchDocument AddSearchDocument(SearchDocument searchDocument) - { - using var context = _dbContextFactory.CreateDbContext(); - context.SearchDocument.Add(searchDocument); - - if(searchDocument.Properties != null && searchDocument.Properties.Any()) - { - foreach(var property in searchDocument.Properties) - { - property.SearchDocumentId = searchDocument.SearchDocumentId; - context.SearchDocumentProperty.Add(property); - } - } - - if (searchDocument.Tags != null && searchDocument.Tags.Any()) - { - foreach (var tag in searchDocument.Tags) - { - tag.SearchDocumentId = searchDocument.SearchDocumentId; - context.SearchDocumentTag.Add(tag); - } - } - - context.SaveChanges(); - - return searchDocument; - } - - public void DeleteSearchDocument(int searchDocumentId) - { - using var db = _dbContextFactory.CreateDbContext(); - var searchDocument = db.SearchDocument.Find(searchDocumentId); - db.SearchDocument.Remove(searchDocument); - db.SaveChanges(); - } - - public void DeleteSearchDocument(string indexerName, int entryId) - { - using var db = _dbContextFactory.CreateDbContext(); - var searchDocument = db.SearchDocument.FirstOrDefault(i => i.IndexerName == indexerName && i.EntryId == entryId); - if(searchDocument != null) - { - db.SearchDocument.Remove(searchDocument); - db.SaveChanges(); - } - } - - public void DeleteSearchDocument(string uniqueKey) - { - using var db = _dbContextFactory.CreateDbContext(); - var searchDocument = db.SearchDocument.FirstOrDefault(i => (i.IndexerName + ":" + i.EntryId) == uniqueKey); - if (searchDocument != null) - { - db.SearchDocument.Remove(searchDocument); - db.SaveChanges(); - } - } - - public void DeleteAllSearchDocuments() - { - using var db = _dbContextFactory.CreateDbContext(); - db.SearchDocument.RemoveRange(db.SearchDocument); - db.SaveChanges(); - } - } -} diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs index caa9fce6..8fd9aab0 100644 --- a/Oqtane.Server/Services/SearchService.cs +++ b/Oqtane.Server/Services/SearchService.cs @@ -1,23 +1,30 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Threading.Tasks; +using HtmlAgilityPack; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Oqtane.Infrastructure; using Oqtane.Models; using Oqtane.Repository; +using Oqtane.Security; using Oqtane.Shared; namespace Oqtane.Services { public class SearchService : ISearchService { + private const string SearchProviderSettingName = "SearchProvider"; + private const string SearchEnabledSettingName = "SearchEnabled"; + private readonly IServiceProvider _serviceProvider; private readonly ITenantManager _tenantManager; private readonly IAliasRepository _aliasRepository; private readonly ISettingRepository _settingRepository; + private readonly IPermissionRepository _permissionRepository; private readonly ILogger _logger; private readonly IMemoryCache _cache; @@ -26,12 +33,14 @@ namespace Oqtane.Services ITenantManager tenantManager, IAliasRepository aliasRepository, ISettingRepository settingRepository, + IPermissionRepository permissionRepository, ILogger logger, IMemoryCache cache) { _tenantManager = tenantManager; _aliasRepository = aliasRepository; _settingRepository = settingRepository; + _permissionRepository = permissionRepository; _serviceProvider = serviceProvider; _logger = logger; _cache = cache; @@ -68,8 +77,8 @@ namespace Oqtane.Services { _logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}"); - var count = searchIndexManager.IndexDocuments(siteId, startTime, SaveIndexDocuments, handleError); - logNote($"Search: Indexer {searchIndexManager.Name} processed {count} documents.
"); + var count = searchIndexManager.IndexContent(siteId, startTime, SaveSearchContent, handleError); + logNote($"Search: Indexer {searchIndexManager.Name} processed {count} search content.
"); _logger.LogDebug($"Search: End Index {searchIndexManager.Name}"); } @@ -79,7 +88,7 @@ namespace Oqtane.Services public async Task SearchAsync(SearchQuery searchQuery) { var searchProvider = GetSearchProvider(searchQuery.SiteId); - var searchResults = await searchProvider.SearchAsync(searchQuery, HasViewPermission); + var searchResults = await searchProvider.SearchAsync(searchQuery, Visible); //generate the document url if it's not set. foreach (var result in searchResults.Results) @@ -108,7 +117,7 @@ namespace Oqtane.Services private string GetSearchProviderSetting(int siteId) { - var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchProviderSettingName); + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchProviderSettingName); if(!string.IsNullOrEmpty(setting?.SettingValue)) { return setting.SettingValue; @@ -119,7 +128,7 @@ namespace Oqtane.Services private bool SearchEnabled(int siteId) { - var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchEnabledSettingName); + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchEnabledSettingName); if (!string.IsNullOrEmpty(setting?.SettingValue)) { return bool.TryParse(setting.SettingValue, out bool enabled) && enabled; @@ -167,21 +176,22 @@ namespace Oqtane.Services return managers.ToList(); } - private void SaveIndexDocuments(IList searchDocuments) + private void SaveSearchContent(IList searchContentList) { - if(searchDocuments.Any()) + if(searchContentList.Any()) { - var searchProvider = GetSearchProvider(searchDocuments.First().SiteId); + var searchProvider = GetSearchProvider(searchContentList.First().SiteId); - foreach (var searchDocument in searchDocuments) + foreach (var searchContent in searchContentList) { try { - searchProvider.SaveDocument(searchDocument); + CleanSearchContent(searchContent); + searchProvider.SaveSearchContent(searchContent); } catch(Exception ex) { - _logger.LogError(ex, $"Search: Save search document {searchDocument.UniqueKey} failed."); + _logger.LogError(ex, $"Search: Save search content {searchContent.UniqueKey} failed."); } } @@ -190,19 +200,30 @@ namespace Oqtane.Services } } - private bool HasViewPermission(SearchDocument searchDocument, SearchQuery searchQuery) + private bool Visible(SearchContent searchContent, SearchQuery searchQuery) { - var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchDocument.IndexerName); + if(!HasViewPermission(searchQuery.SiteId, searchQuery.User, searchContent.EntityName, searchContent.EntityId)) + { + return false; + } + + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchContent.EntityName); if (searchResultManager != null) { - return searchResultManager.Visible(searchDocument, searchQuery); + return searchResultManager.Visible(searchContent, searchQuery); } return true; } + private bool HasViewPermission(int siteId, User user, string entityName, int entityId) + { + var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList(); + return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions); + } + private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery) { - var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.IndexerName); + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.EntityName); if(searchResultManager != null) { return searchResultManager.GetUrl(result, searchQuery); @@ -210,5 +231,35 @@ namespace Oqtane.Services return string.Empty; } + + private void CleanSearchContent(SearchContent searchContent) + { + searchContent.Title = GetCleanContent(searchContent.Title); + searchContent.Description = GetCleanContent(searchContent.Description); + searchContent.Body = GetCleanContent(searchContent.Body); + searchContent.AdditionalContent = GetCleanContent(searchContent.AdditionalContent); + } + + private string GetCleanContent(string content) + { + if(string.IsNullOrWhiteSpace(content)) + { + return string.Empty; + } + + content = WebUtility.HtmlDecode(content); + + var page = new HtmlDocument(); + page.LoadHtml(content); + + var phrases = page.DocumentNode.Descendants().Where(i => + i.NodeType == HtmlNodeType.Text && + i.ParentNode.Name != "script" && + i.ParentNode.Name != "style" && + !string.IsNullOrEmpty(i.InnerText.Trim()) + ).Select(i => i.InnerText); + + return string.Join(" ", phrases); + } } } diff --git a/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css similarity index 100% rename from Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css rename to Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css diff --git a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs index 9a322ab5..0a4123f8 100644 --- a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs +++ b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs @@ -15,6 +15,6 @@ namespace Oqtane.Services bool IsIndexEnabled(int siteId); - int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError); + int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError); } } diff --git a/Oqtane.Shared/Interfaces/ISearchProvider.cs b/Oqtane.Shared/Interfaces/ISearchProvider.cs index e1e9fb16..cb075d3c 100644 --- a/Oqtane.Shared/Interfaces/ISearchProvider.cs +++ b/Oqtane.Shared/Interfaces/ISearchProvider.cs @@ -11,11 +11,11 @@ namespace Oqtane.Services { string Name { get; } - void SaveDocument(SearchDocument document, bool autoCommit = false); + void SaveSearchContent(SearchContent searchContent, bool autoCommit = false); - void DeleteDocument(string id); + void DeleteSearchContent(string id); - Task SearchAsync(SearchQuery searchQuery, Func validateFunc); + Task SearchAsync(SearchQuery searchQuery, Func validateFunc); bool Optimize(); diff --git a/Oqtane.Shared/Interfaces/ISearchResultManager.cs b/Oqtane.Shared/Interfaces/ISearchResultManager.cs index 2dc459b0..b6f6cd32 100644 --- a/Oqtane.Shared/Interfaces/ISearchResultManager.cs +++ b/Oqtane.Shared/Interfaces/ISearchResultManager.cs @@ -7,7 +7,7 @@ namespace Oqtane.Services { string Name { get; } - bool Visible(SearchDocument searchResult, SearchQuery searchQuery); + bool Visible(SearchContent searchResult, SearchQuery searchQuery); string GetUrl(SearchResult searchResult, SearchQuery searchQuery); } diff --git a/Oqtane.Shared/Interfaces/IModuleSearch.cs b/Oqtane.Shared/Interfaces/ISearchable.cs similarity index 58% rename from Oqtane.Shared/Interfaces/IModuleSearch.cs rename to Oqtane.Shared/Interfaces/ISearchable.cs index af94a300..c0a4c00e 100644 --- a/Oqtane.Shared/Interfaces/IModuleSearch.cs +++ b/Oqtane.Shared/Interfaces/ISearchable.cs @@ -7,8 +7,8 @@ using Oqtane.Models; namespace Oqtane.Interfaces { - public interface IModuleSearch + public interface ISearchable { - public IList GetSearchDocuments(Module module, DateTime startTime); + public IList GetSearchContentList(Module module, DateTime startTime); } } diff --git a/Oqtane.Shared/Models/SearchDocument.cs b/Oqtane.Shared/Models/SearchContent.cs similarity index 57% rename from Oqtane.Shared/Models/SearchDocument.cs rename to Oqtane.Shared/Models/SearchContent.cs index 67161b3b..36f4c664 100644 --- a/Oqtane.Shared/Models/SearchDocument.cs +++ b/Oqtane.Shared/Models/SearchContent.cs @@ -4,16 +4,16 @@ using System.ComponentModel.DataAnnotations.Schema; using System.Text.Json; namespace Oqtane.Models { - public class SearchDocument : ModelBase + public class SearchContent : ModelBase { - public int SearchDocumentId { get; set; } + public int SearchContentId { get; set; } [NotMapped] - public string UniqueKey => $"{IndexerName}:{EntryId}"; + public string UniqueKey => $"{EntityName}:{EntityId}"; - public int EntryId { get; set; } + public string EntityName { get; set; } - public string IndexerName { get; set; } + public int EntityId { get; set; } public int SiteId { get; set; } @@ -27,15 +27,13 @@ namespace Oqtane.Models public DateTime ModifiedTime { get; set; } - public bool IsActive { get; set; } + public bool IsActive { get; set; } = true; public string AdditionalContent { get; set; } - public string LanguageCode { get; set; } + public IList Properties { get; set; } - public IList Tags { get; set; } - - public IList Properties { get; set; } + public IList Words { get; set; } public override string ToString() { diff --git a/Oqtane.Shared/Models/SearchDocumentProperty.cs b/Oqtane.Shared/Models/SearchContentProperty.cs similarity index 74% rename from Oqtane.Shared/Models/SearchDocumentProperty.cs rename to Oqtane.Shared/Models/SearchContentProperty.cs index 4febae43..0c2b7f9b 100644 --- a/Oqtane.Shared/Models/SearchDocumentProperty.cs +++ b/Oqtane.Shared/Models/SearchContentProperty.cs @@ -3,12 +3,12 @@ using Microsoft.EntityFrameworkCore; namespace Oqtane.Models { - public class SearchDocumentProperty + public class SearchContentProperty { [Key] public int PropertyId { get; set; } - public int SearchDocumentId { get; set; } + public int SearchContentId { get; set; } public string Name { get; set; } diff --git a/Oqtane.Shared/Models/SearchContentWordSource.cs b/Oqtane.Shared/Models/SearchContentWordSource.cs new file mode 100644 index 00000000..ec814757 --- /dev/null +++ b/Oqtane.Shared/Models/SearchContentWordSource.cs @@ -0,0 +1,12 @@ +using System.ComponentModel.DataAnnotations; + +namespace Oqtane.Models +{ + public class SearchContentWordSource + { + [Key] + public int WordSourceId { get; set; } + + public string Word { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchContentWords.cs b/Oqtane.Shared/Models/SearchContentWords.cs new file mode 100644 index 00000000..c7630815 --- /dev/null +++ b/Oqtane.Shared/Models/SearchContentWords.cs @@ -0,0 +1,19 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace Oqtane.Models +{ + public class SearchContentWords + { + [Key] + public int WordId { get; set; } + + public int SearchContentId { get; set; } + + public int WordSourceId { get; set; } + + public int Count { get; set; } + + public SearchContentWordSource WordSource { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchDocumentTag.cs b/Oqtane.Shared/Models/SearchDocumentTag.cs deleted file mode 100644 index ee929efc..00000000 --- a/Oqtane.Shared/Models/SearchDocumentTag.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace Oqtane.Models -{ - public class SearchDocumentTag - { - [Key] - public int TagId { get; set; } - - public int SearchDocumentId { get; set; } - - public string Tag { get; set; } - } -} diff --git a/Oqtane.Shared/Models/SearchQuery.cs b/Oqtane.Shared/Models/SearchQuery.cs index f148252d..cf03e8c1 100644 --- a/Oqtane.Shared/Models/SearchQuery.cs +++ b/Oqtane.Shared/Models/SearchQuery.cs @@ -14,14 +14,12 @@ namespace Oqtane.Models public string Keywords { get; set; } - public IList Sources { get; set; } = new List(); + public IList EntityNames { get; set; } = new List(); public DateTime BeginModifiedTimeUtc { get; set; } public DateTime EndModifiedTimeUtc { get; set; } - public IList Tags { get; set; } = new List(); - public IDictionary Properties { get; set; } = new Dictionary(); public int PageIndex { get; set; } diff --git a/Oqtane.Shared/Models/SearchResult.cs b/Oqtane.Shared/Models/SearchResult.cs index d13cafa1..dd878d42 100644 --- a/Oqtane.Shared/Models/SearchResult.cs +++ b/Oqtane.Shared/Models/SearchResult.cs @@ -1,6 +1,6 @@ namespace Oqtane.Models { - public class SearchResult : SearchDocument + public class SearchResult : SearchContent { public float Score { get; set; } diff --git a/Oqtane.Shared/Shared/Constants.cs b/Oqtane.Shared/Shared/Constants.cs index f1a14cd4..979f0872 100644 --- a/Oqtane.Shared/Shared/Constants.cs +++ b/Oqtane.Shared/Shared/Constants.cs @@ -77,22 +77,10 @@ namespace Oqtane.Shared public static readonly string VisitorCookiePrefix = "APP_VISITOR_"; - public const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; - public const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; - public const string SearchResultManagersCacheName = "SearchResultManagers"; - public const int SearchDefaultPageSize = 10; + public const string DefaultSearchProviderName = "Database"; public const string SearchPageIdPropertyName = "PageId"; public const string SearchModuleIdPropertyName = "ModuleId"; - public const string DefaultSearchProviderName = "Database"; - public const string SearchProviderSettingName = "SearchProvider"; - public const string SearchEnabledSettingName = "SearchEnabled"; - - public const string ModuleSearchIndexManagerName = "Module"; - public const string PageSearchIndexManagerName = "Page"; - - public const int PageSearchIndexManagerPriority = 100; - public const int ModuleSearchIndexManagerPriority = 200; - + // Obsolete constants const string RoleObsoleteMessage = "Use the corresponding member from Oqtane.Shared.RoleNames"; diff --git a/Oqtane.Shared/Shared/SearchUtils.cs b/Oqtane.Shared/Shared/SearchUtils.cs index b7116d4a..745770a5 100644 --- a/Oqtane.Shared/Shared/SearchUtils.cs +++ b/Oqtane.Shared/Shared/SearchUtils.cs @@ -1,40 +1,14 @@ -using System.Collections; using System.Collections.Generic; -using System.Net; -using System.Text.RegularExpressions; namespace Oqtane.Shared { public sealed class SearchUtils { - private const string PunctuationMatch = "[~!#\\$%\\^&*\\(\\)-+=\\{\\[\\}\\]\\|;:\\x22'<,>\\.\\?\\\\\\t\\r\\v\\f\\n]"; - private static readonly Regex _stripWhiteSpaceRegex = new Regex("\\s+", RegexOptions.Compiled); - private static readonly Regex _stripTagsRegex = new Regex("<[^<>]*>", RegexOptions.Compiled); - private static readonly Regex _afterRegEx = new Regex(PunctuationMatch + "\\s", RegexOptions.Compiled); - private static readonly Regex _beforeRegEx = new Regex("\\s" + PunctuationMatch, RegexOptions.Compiled); + private static readonly IList _systemPages; - public static string Clean(string html, bool removePunctuation) + static SearchUtils() { - if (string.IsNullOrWhiteSpace(html)) - { - return string.Empty; - } - - if (html.Contains("<")) - { - html = WebUtility.HtmlDecode(html); - } - - html = StripTags(html, true); - html = WebUtility.HtmlDecode(html); - - if (removePunctuation) - { - html = StripPunctuation(html, true); - html = StripWhiteSpace(html, true); - } - - return html; + _systemPages = new List { "login", "register", "profile", "404", "search" }; } public static IList GetKeywordsList(string keywords) @@ -54,42 +28,9 @@ namespace Oqtane.Shared return keywordsList; } - private static string StripTags(string html, bool retainSpace) + public static bool IsSystemPage(Models.Page page) { - return _stripTagsRegex.Replace(html, retainSpace ? " " : string.Empty); - } - - private static string StripPunctuation(string html, bool retainSpace) - { - if (string.IsNullOrWhiteSpace(html)) - { - return string.Empty; - } - - string retHTML = html + " "; - - var repString = retainSpace ? " " : string.Empty; - while (_beforeRegEx.IsMatch(retHTML)) - { - retHTML = _beforeRegEx.Replace(retHTML, repString); - } - - while (_afterRegEx.IsMatch(retHTML)) - { - retHTML = _afterRegEx.Replace(retHTML, repString); - } - - return retHTML.Trim('"'); - } - - private static string StripWhiteSpace(string html, bool retainSpace) - { - if (string.IsNullOrWhiteSpace(html)) - { - return string.Empty; - } - - return _stripWhiteSpaceRegex.Replace(html, retainSpace ? " " : string.Empty); + return page.Path.Contains("admin") || _systemPages.Contains(page.Path); } } } From 790fc88e47516bea45bd9f5ac873914e224a57c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 4 Jun 2024 17:50:29 +0800 Subject: [PATCH 3/5] using correct module id value. --- Oqtane.Client/Modules/Admin/SearchResults/Index.razor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oqtane.Client/Modules/Admin/SearchResults/Index.razor b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor index 2cbc3098..3b25090e 100644 --- a/Oqtane.Client/Modules/Admin/SearchResults/Index.razor +++ b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor @@ -127,7 +127,7 @@ PageSize = int.MaxValue }; - _searchResults = await SearchResultsService.SearchAsync(PageState.ModuleId, searchQuery); + _searchResults = await SearchResultsService.SearchAsync(ModuleState.ModuleId, searchQuery); _loading = false; StateHasChanged(); From d9d917e267e675cae448c41fba1050fbc500005e Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 4 Jun 2024 21:09:25 +0800 Subject: [PATCH 4/5] set search result page path to be parameter. --- Oqtane.Client/Themes/Controls/Theme/Search.razor | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Oqtane.Client/Themes/Controls/Theme/Search.razor b/Oqtane.Client/Themes/Controls/Theme/Search.razor index fc4980ef..a782bdaf 100644 --- a/Oqtane.Client/Themes/Controls/Theme/Search.razor +++ b/Oqtane.Client/Themes/Controls/Theme/Search.razor @@ -26,17 +26,21 @@ @code { - private const string SearchResultPagePath = "search"; - private Page _searchResultsPage; private string _keywords = ""; [Parameter] public string CssClass { get; set; } + [Parameter] + public string SearchResultPagePath { get; set; } = "search"; + protected override void OnInitialized() { - _searchResultsPage = PageState.Pages.FirstOrDefault(i => i.Path == SearchResultPagePath); + if(!string.IsNullOrEmpty(SearchResultPagePath)) + { + _searchResultsPage = PageState.Pages.FirstOrDefault(i => i.Path == SearchResultPagePath); + } } protected override void OnParametersSet() From e1cdc7b387257be0674c500daa4d57e7da85b394 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 4 Jun 2024 21:57:50 +0800 Subject: [PATCH 5/5] return the words count to calculate the ranking. --- .../Providers/DatabaseSearchProvider.cs | 77 +++++++++++-------- .../Repository/SearchContentRepository.cs | 2 + Oqtane.Server/Services/SearchService.cs | 31 -------- Oqtane.Shared/Shared/SearchUtils.cs | 2 +- 4 files changed, 48 insertions(+), 64 deletions(-) diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs index 74884934..adf2cee8 100644 --- a/Oqtane.Server/Providers/DatabaseSearchProvider.cs +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -18,10 +18,6 @@ namespace Oqtane.Providers { private readonly ISearchContentRepository _searchContentRepository; - private const float TitleBoost = 100f; - private const float DescriptionBoost = 10f; - private const float BodyBoost = 10f; - private const float AdditionalContentBoost = 5f; private const string IgnoreWords = "the,be,to,of,and,a,i,in,that,have,it,for,not,on,with,he,as,you,do,at,this,but,his,by,from,they,we,say,her,she,or,an,will,my,one,all,would,there,their,what,so,up,out,if,about,who,get,which,go,me,when,make,can,like,time,no,just,him,know,take,people,into,year,your,good,some,could,them,see,other,than,then,now,look,only,come,its,over,think,also,back,after,use,two,how,our,work,first,well,way,even,new,want,because,any,these,give,day,most,us"; private const int WordMinLength = 3; public string Name => Constants.DefaultSearchProviderName; @@ -55,6 +51,9 @@ namespace Oqtane.Providers //remove exist document _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId); + //clean the search content to remove html tags + CleanSearchContent(searchContent); + _searchContentRepository.AddSearchContent(searchContent); //save the index words @@ -152,10 +151,7 @@ namespace Oqtane.Providers var score = 0f; foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) { - score += Regex.Matches(searchContent.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost; - score += Regex.Matches(searchContent.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost; - score += Regex.Matches(searchContent.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost; - score += Regex.Matches(searchContent.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost; + score += searchContent.Words.Where(i => i.WordSource.Word.StartsWith(keyword)).Sum(i => i.Count); } return score / 100; @@ -241,37 +237,24 @@ namespace Oqtane.Providers private static Dictionary GetWords(string content, int minLength) { - content = WebUtility.HtmlDecode(content); + content = FormatText(content); var words = new Dictionary(); var ignoreWords = IgnoreWords.Split(','); - var page = new HtmlDocument(); - page.LoadHtml(content); - - var phrases = page.DocumentNode.Descendants().Where(i => - i.NodeType == HtmlNodeType.Text && - i.ParentNode.Name != "script" && - i.ParentNode.Name != "style" && - !string.IsNullOrEmpty(i.InnerText.Trim()) - ).Select(i => FormatText(i.InnerText)); - - foreach (var phrase in phrases) + if (!string.IsNullOrEmpty(content)) { - if (!string.IsNullOrEmpty(phrase)) + foreach (var word in content.Split(' ')) { - foreach (var word in phrase.Split(' ')) + if (word.Length >= minLength && !ignoreWords.Contains(word)) { - if (word.Length >= minLength && !ignoreWords.Contains(word)) + if (!words.ContainsKey(word)) { - if (!words.ContainsKey(word)) - { - words.Add(word, 1); - } - else - { - words[word] += 1; - } + words.Add(word, 1); + } + else + { + words[word] += 1; } } } @@ -288,8 +271,38 @@ namespace Oqtane.Providers text = text.Replace(punctuation, ' '); } text = text.Replace(" ", " ").ToLower().Trim(); - return text; + return text; + } + + private void CleanSearchContent(SearchContent searchContent) + { + searchContent.Title = GetCleanContent(searchContent.Title); + searchContent.Description = GetCleanContent(searchContent.Description); + searchContent.Body = GetCleanContent(searchContent.Body); + searchContent.AdditionalContent = GetCleanContent(searchContent.AdditionalContent); + } + + private string GetCleanContent(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return string.Empty; + } + + content = WebUtility.HtmlDecode(content); + + var page = new HtmlDocument(); + page.LoadHtml(content); + + var phrases = page.DocumentNode.Descendants().Where(i => + i.NodeType == HtmlNodeType.Text && + i.ParentNode.Name != "script" && + i.ParentNode.Name != "style" && + !string.IsNullOrEmpty(i.InnerText.Trim()) + ).Select(i => i.InnerText); + + return string.Join(" ", phrases); } } } diff --git a/Oqtane.Server/Repository/SearchContentRepository.cs b/Oqtane.Server/Repository/SearchContentRepository.cs index c1577d81..67ecfcf7 100644 --- a/Oqtane.Server/Repository/SearchContentRepository.cs +++ b/Oqtane.Server/Repository/SearchContentRepository.cs @@ -22,6 +22,8 @@ namespace Oqtane.Repository using var db = _dbContextFactory.CreateDbContext(); var searchContentList = db.SearchContent.AsNoTracking() .Include(i => i.Properties) + .Include(i => i.Words) + .ThenInclude(w => w.WordSource) .Where(i => i.SiteId == searchQuery.SiteId && i.IsActive); if (searchQuery.EntityNames != null && searchQuery.EntityNames.Any()) diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs index 8fd9aab0..8d3fe4ef 100644 --- a/Oqtane.Server/Services/SearchService.cs +++ b/Oqtane.Server/Services/SearchService.cs @@ -186,7 +186,6 @@ namespace Oqtane.Services { try { - CleanSearchContent(searchContent); searchProvider.SaveSearchContent(searchContent); } catch(Exception ex) @@ -231,35 +230,5 @@ namespace Oqtane.Services return string.Empty; } - - private void CleanSearchContent(SearchContent searchContent) - { - searchContent.Title = GetCleanContent(searchContent.Title); - searchContent.Description = GetCleanContent(searchContent.Description); - searchContent.Body = GetCleanContent(searchContent.Body); - searchContent.AdditionalContent = GetCleanContent(searchContent.AdditionalContent); - } - - private string GetCleanContent(string content) - { - if(string.IsNullOrWhiteSpace(content)) - { - return string.Empty; - } - - content = WebUtility.HtmlDecode(content); - - var page = new HtmlDocument(); - page.LoadHtml(content); - - var phrases = page.DocumentNode.Descendants().Where(i => - i.NodeType == HtmlNodeType.Text && - i.ParentNode.Name != "script" && - i.ParentNode.Name != "style" && - !string.IsNullOrEmpty(i.InnerText.Trim()) - ).Select(i => i.InnerText); - - return string.Join(" ", phrases); - } } } diff --git a/Oqtane.Shared/Shared/SearchUtils.cs b/Oqtane.Shared/Shared/SearchUtils.cs index 745770a5..d49c932d 100644 --- a/Oqtane.Shared/Shared/SearchUtils.cs +++ b/Oqtane.Shared/Shared/SearchUtils.cs @@ -20,7 +20,7 @@ namespace Oqtane.Shared { if (!string.IsNullOrWhiteSpace(keyword.Trim())) { - keywordsList.Add(keyword.Trim()); + keywordsList.Add(keyword.Trim().ToLower()); } } }