From 9d85ca07f47c9ee5abad20190868a720e66b771d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 3 Jun 2024 21:19:42 +0800 Subject: [PATCH] #4303: add search function. --- .../Modules/SearchResults/Index.razor | 147 ++++++++++++ .../Modules/SearchResults/ModuleInfo.cs | 25 ++ .../Services/ISearchResultsService.cs | 13 ++ .../Services/SearchResultsService.cs | 23 ++ .../Modules/SearchResults/Settings.razor | 46 ++++ .../Modules/SearchResults/Index.resx | 156 +++++++++++++ .../Modules/SearchResults/Settings.resx | 123 ++++++++++ .../Resources/Themes/Controls/Search.resx | 126 +++++++++++ .../Themes/BlazorTheme/Themes/Default.razor | 4 +- .../Themes/Controls/Theme/Search.razor | 57 +++++ .../Themes/OqtaneTheme/Themes/Default.razor | 7 +- .../OqtaneServiceCollectionExtensions.cs | 4 + .../Infrastructure/Jobs/SearchIndexJob.cs | 88 +++++++ .../SiteTemplates/DefaultSiteTemplate.cs | 24 ++ .../Search/ModuleSearchIndexManager.cs | 127 +++++++++++ .../Search/ModuleSearchResultManager.cs | 69 ++++++ .../Managers/Search/PageSearchIndexManager.cs | 94 ++++++++ .../Search/PageSearchResultManager.cs | 46 ++++ .../Managers/Search/SearchIndexManagerBase.cs | 34 +++ .../SearchDocumentEntityBuilder.cs | 64 ++++++ .../SearchDocumentPropertyEntityBuilder.cs | 40 ++++ .../SearchDocumentTagEntityBuilder.cs | 37 +++ .../Tenant/05020001_AddSearchTables.cs | 42 ++++ .../HtmlText/Manager/HtmlTextManager.cs | 34 ++- .../Controllers/SearchResultsController.cs | 43 ++++ .../Services/SearchResultsService.cs | 36 +++ .../SearchResults/Startup/ServerStartup.cs | 24 ++ .../Providers/DatabaseSearchProvider.cs | 202 +++++++++++++++++ .../Repository/Context/TenantDBContext.cs | 3 + .../Interfaces/ISearchDocumentRepository.cs | 17 ++ .../Repository/SearchDocumentRepository.cs | 136 +++++++++++ Oqtane.Server/Services/SearchService.cs | 214 ++++++++++++++++++ .../Oqtane.Modules.SearchResults/Module.css | 3 + .../Oqtane.Themes.OqtaneTheme/Theme.css | 4 + Oqtane.Server/wwwroot/css/app.css | 14 ++ Oqtane.Shared/Enums/SearchSortDirections.cs | 10 + Oqtane.Shared/Enums/SearchSortFields.cs | 11 + Oqtane.Shared/Interfaces/IModuleSearch.cs | 14 ++ .../Interfaces/ISearchIndexManager.cs | 20 ++ Oqtane.Shared/Interfaces/ISearchProvider.cs | 26 +++ .../Interfaces/ISearchResultManager.cs | 14 ++ Oqtane.Shared/Interfaces/ISearchService.cs | 15 ++ Oqtane.Shared/Models/SearchDocument.cs | 45 ++++ .../Models/SearchDocumentProperty.cs | 17 ++ Oqtane.Shared/Models/SearchDocumentTag.cs | 14 ++ Oqtane.Shared/Models/SearchQuery.cs | 37 +++ Oqtane.Shared/Models/SearchResult.cs | 11 + Oqtane.Shared/Models/SearchResults.cs | 11 + Oqtane.Shared/Shared/Constants.cs | 16 ++ Oqtane.Shared/Shared/SearchUtils.cs | 95 ++++++++ 50 files changed, 2478 insertions(+), 4 deletions(-) create mode 100644 Oqtane.Client/Modules/SearchResults/Index.razor create mode 100644 Oqtane.Client/Modules/SearchResults/ModuleInfo.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs create mode 100644 Oqtane.Client/Modules/SearchResults/Settings.razor create mode 100644 Oqtane.Client/Resources/Modules/SearchResults/Index.resx create mode 100644 Oqtane.Client/Resources/Modules/SearchResults/Settings.resx create mode 100644 Oqtane.Client/Resources/Themes/Controls/Search.resx create mode 100644 Oqtane.Client/Themes/Controls/Theme/Search.razor create mode 100644 Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs create mode 100644 Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs create mode 100644 Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs create mode 100644 Oqtane.Server/Managers/Search/PageSearchIndexManager.cs create mode 100644 Oqtane.Server/Managers/Search/PageSearchResultManager.cs create mode 100644 Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs create mode 100644 Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs create mode 100644 Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs create mode 100644 Oqtane.Server/Providers/DatabaseSearchProvider.cs create mode 100644 Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs create mode 100644 Oqtane.Server/Repository/SearchDocumentRepository.cs create mode 100644 Oqtane.Server/Services/SearchService.cs create mode 100644 Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css create mode 100644 Oqtane.Shared/Enums/SearchSortDirections.cs create mode 100644 Oqtane.Shared/Enums/SearchSortFields.cs create mode 100644 Oqtane.Shared/Interfaces/IModuleSearch.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchIndexManager.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchProvider.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchResultManager.cs create mode 100644 Oqtane.Shared/Interfaces/ISearchService.cs create mode 100644 Oqtane.Shared/Models/SearchDocument.cs create mode 100644 Oqtane.Shared/Models/SearchDocumentProperty.cs create mode 100644 Oqtane.Shared/Models/SearchDocumentTag.cs create mode 100644 Oqtane.Shared/Models/SearchQuery.cs create mode 100644 Oqtane.Shared/Models/SearchResult.cs create mode 100644 Oqtane.Shared/Models/SearchResults.cs create mode 100644 Oqtane.Shared/Shared/SearchUtils.cs diff --git a/Oqtane.Client/Modules/SearchResults/Index.razor b/Oqtane.Client/Modules/SearchResults/Index.razor new file mode 100644 index 00000000..8ae0c087 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Index.razor @@ -0,0 +1,147 @@ +@using Oqtane.Modules.SearchResults.Services +@namespace Oqtane.Modules.SearchResults +@inherits ModuleBase +@inject ISearchResultsService SearchResultsService +@inject IStringLocalizer Localizer + +
+
+
+
+ @Localizer["SearchPrefix"] + + +
+
+
+
+
+ @if (_loading) + { +
+ } + else + { + @if (_searchResults != null && _searchResults.Results != null) + { + if (_searchResults.Results.Any()) + { + + +
+

@context.Title

+
@context.Url
+

@((MarkupString)context.Snippet)

+
+ + + } + else + { + + } + } +
+ } +
+
+
+@code { + private SearchSortDirections _searchSortDirection = SearchSortDirections.Descending; //default sort by + private SearchSortFields _searchSortField = SearchSortFields.Relevance; + private string _keywords; + private bool _loading; + private SearchResults _searchResults; + private int _currentPage = 0; + private int _pageSize = Constants.SearchDefaultPageSize; + private int _displayPages = 7; + + protected override async Task OnInitializedAsync() + { + if (ModuleState.Settings.ContainsKey("PageSize")) + { + _pageSize = int.Parse(ModuleState.Settings["PageSize"]); + } + + if (PageState.QueryString.ContainsKey("s")) + { + _keywords = PageState.QueryString["s"]; + } + + if (PageState.QueryString.ContainsKey("p")) + { + _currentPage = Convert.ToInt32(PageState.QueryString["p"]); + if (_currentPage < 1) + { + _currentPage = 1; + } + } + + if (!string.IsNullOrEmpty(_keywords)) + { + await PerformSearch(); + } + } + + private async Task KeywordsChanged(KeyboardEventArgs e) + { + if (e.Code == "Enter" || e.Code == "NumpadEnter") + { + if (!string.IsNullOrEmpty(_keywords)) + { + await Search(); + } + } + } + + private async Task Search() + { + if (string.IsNullOrEmpty(_keywords)) + { + AddModuleMessage(Localizer["MissingKeywords"], MessageType.Warning); + } + else + { + ClearModuleMessage(); + + + _currentPage = 0; + await PerformSearch(); + } + } + + private async Task PerformSearch() + { + _loading = true; + StateHasChanged(); + + var searchQuery = new SearchQuery + { + SiteId = PageState.Site.SiteId, + Alias = PageState.Alias, + User = PageState.User, + Keywords = _keywords, + SortDirection = _searchSortDirection, + SortField = _searchSortField, + PageIndex = 0, + PageSize = int.MaxValue + }; + + _searchResults = await SearchResultsService.SearchAsync(PageState.ModuleId, searchQuery); + + _loading = false; + StateHasChanged(); + } +} \ No newline at end of file diff --git a/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs b/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs new file mode 100644 index 00000000..ae4bc7d2 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/ModuleInfo.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class ModuleInfo : IModule + { + public ModuleDefinition ModuleDefinition => new ModuleDefinition + { + Name = "Search Results", + Description = "Display Search Results", + Version = "1.0.0", + ServerManagerType = "", + ReleaseVersions = "1.0.0", + SettingsType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client", + Resources = new List() + { + new Resource { ResourceType = ResourceType.Stylesheet, Url = "~/Module.css" } + } + }; + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs b/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs new file mode 100644 index 00000000..2f1c1308 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Services/ISearchResultsService.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public interface ISearchResultsService + { + Task SearchAsync(int moduleId, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs new file mode 100644 index 00000000..3b241ea4 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Services/SearchResultsService.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class SearchResultsService : ServiceBase, ISearchResultsService, IClientService + { + public SearchResultsService(HttpClient http, SiteState siteState) : base(http, siteState) {} + + private string ApiUrl => CreateApiUrl("SearchResults"); + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + return await PostJsonAsync(CreateAuthorizationPolicyUrl(ApiUrl, EntityNames.Module, moduleId), searchQuery); + } + } +} diff --git a/Oqtane.Client/Modules/SearchResults/Settings.razor b/Oqtane.Client/Modules/SearchResults/Settings.razor new file mode 100644 index 00000000..b38d2946 --- /dev/null +++ b/Oqtane.Client/Modules/SearchResults/Settings.razor @@ -0,0 +1,46 @@ +@namespace Oqtane.Modules.SearchResults +@inherits ModuleBase +@inject ISettingService SettingService +@implements Oqtane.Interfaces.ISettingsControl +@inject IStringLocalizer Localizer +@inject IStringLocalizer SharedLocalizer + +
+
+ +
+ +
+
+
+ +@code { + private string resourceType = "Oqtane.Modules.SearchResults.Settings, Oqtane.Client"; // for localization + private string _pageSize; + + protected override void OnInitialized() + { + try + { + _pageSize = SettingService.GetSetting(ModuleState.Settings, "PageSize", Constants.SearchDefaultPageSize.ToString()); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } + + public async Task UpdateSettings() + { + try + { + var settings = await SettingService.GetModuleSettingsAsync(ModuleState.ModuleId); + settings = SettingService.SetSetting(settings, "PageSize", _pageSize); + await SettingService.UpdateModuleSettingsAsync(settings, ModuleState.ModuleId); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } +} diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Index.resx b/Oqtane.Client/Resources/Modules/SearchResults/Index.resx new file mode 100644 index 00000000..bd07f3f8 --- /dev/null +++ b/Oqtane.Client/Resources/Modules/SearchResults/Index.resx @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Ascending + + + Descending + + + Please provide the search keywords. + + + Modification Time + + + No results found. + + + Relevance + + + Search + + + Search + + + Search: + + + Sort Direction + + + Sort By + + + Title + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx b/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx new file mode 100644 index 00000000..1c48707d --- /dev/null +++ b/Oqtane.Client/Resources/Modules/SearchResults/Settings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Page Size + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Themes/Controls/Search.resx b/Oqtane.Client/Resources/Themes/Controls/Search.resx new file mode 100644 index 00000000..c2d2b4ee --- /dev/null +++ b/Oqtane.Client/Resources/Themes/Controls/Search.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Search + + + Search + + \ No newline at end of file diff --git a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor index 553d97d1..2e0da4d3 100644 --- a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor @@ -9,7 +9,9 @@
diff --git a/Oqtane.Client/Themes/Controls/Theme/Search.razor b/Oqtane.Client/Themes/Controls/Theme/Search.razor new file mode 100644 index 00000000..dbd28b40 --- /dev/null +++ b/Oqtane.Client/Themes/Controls/Theme/Search.razor @@ -0,0 +1,57 @@ +@namespace Oqtane.Themes.Controls +@using System.Net +@using Microsoft.AspNetCore.Http +@inherits ThemeControlBase +@inject IStringLocalizer Localizer +@inject NavigationManager NavigationManager +@inject IHttpContextAccessor HttpContext + +@if (_searchResultsPage != null) +{ + +
+ + + +
+ +} + + + +@code { + private const string SearchResultPagePath = "search-results"; + + private Page _searchResultsPage; + private string _keywords = ""; + + [Parameter] + public string CssClass { get; set; } + + protected override void OnInitialized() + { + _searchResultsPage = PageState.Pages.FirstOrDefault(i => i.Path == SearchResultPagePath); + } + + protected override void OnParametersSet() + { + } + + private void PerformSearch() + { + var keywords = HttpContext.HttpContext.Request.Form["keywords"]; + if (!string.IsNullOrEmpty(keywords) && _searchResultsPage != null) + { + var url = NavigateUrl(_searchResultsPage.Path, $"s={keywords}"); + NavigationManager.NavigateTo(url); + } + } +} + + diff --git a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor index 261597aa..936c823e 100644 --- a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor @@ -6,7 +6,12 @@
diff --git a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs index 19eb4cbe..a6612866 100644 --- a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs +++ b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs @@ -21,6 +21,7 @@ using Oqtane.Infrastructure; using Oqtane.Infrastructure.Interfaces; using Oqtane.Managers; using Oqtane.Modules; +using Oqtane.Providers; using Oqtane.Repository; using Oqtane.Security; using Oqtane.Services; @@ -97,6 +98,8 @@ namespace Microsoft.Extensions.DependencyInjection services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); return services; } @@ -131,6 +134,7 @@ namespace Microsoft.Extensions.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); // managers services.AddTransient(); diff --git a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs new file mode 100644 index 00000000..e6b7f216 --- /dev/null +++ b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs @@ -0,0 +1,88 @@ +using System; +using System.Linq; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Infrastructure +{ + public class SearchIndexJob : HostedServiceBase + { + public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory) + { + Name = "Search Index Job"; + Frequency = "m"; // run every minute. + Interval = 1; + IsEnabled = true; + } + + public override string ExecuteJob(IServiceProvider provider) + { + // get services + var siteRepository = provider.GetRequiredService(); + var settingRepository = provider.GetRequiredService(); + var logRepository = provider.GetRequiredService(); + var searchService = provider.GetRequiredService(); + + var sites = siteRepository.GetSites().ToList(); + var logs = new StringBuilder(); + + foreach (var site in sites) + { + var startTime = GetSearchStartTime(site.SiteId, settingRepository); + logs.AppendLine($"Search: Begin index site: {site.Name}
"); + var currentTime = DateTime.UtcNow; + + searchService.IndexContent(site.SiteId, startTime, logNote => + { + logs.AppendLine(logNote); + }, handleError => + { + logs.AppendLine(handleError); + }); + + UpdateSearchStartTime(site.SiteId, currentTime, settingRepository); + + logs.AppendLine($"Search: End index site: {site.Name}
"); + } + + return logs.ToString(); + } + + private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + if(setting == null) + { + return null; + } + + return Convert.ToDateTime(setting.SettingValue); + } + + private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName); + if (setting == null) + { + setting = new Setting + { + EntityName = EntityNames.Site, + EntityId = siteId, + SettingName = Constants.SearchIndexStartTimeSettingName, + SettingValue = Convert.ToString(startTime), + }; + + settingRepository.AddSetting(setting); + } + else + { + setting.SettingValue = Convert.ToString(startTime); + settingRepository.UpdateSetting(setting); + } + } + } +} diff --git a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs index 7cf27808..99bb8aaa 100644 --- a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs +++ b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs @@ -133,6 +133,30 @@ namespace Oqtane.SiteTemplates } } }); + _pageTemplates.Add(new PageTemplate + { + Name = "Search Results", + Parent = "", + Order = 7, + Path = "search-results", + Icon = "oi oi-magnifying-glass", + IsNavigation = false, + IsPersonalizable = false, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + }, + PageTemplateModules = new List { + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + } + } + } + }); if (System.IO.File.Exists(Path.Combine(_environment.WebRootPath, "images", "logo-white.png"))) { diff --git a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs new file mode 100644 index 00000000..fec739d2 --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchIndexManager : SearchIndexManagerBase + { + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageModuleRepository _pageModuleRepostory; + private readonly IPageRepository _pageRepository; + + public ModuleSearchIndexManager( + IServiceProvider serviceProvider, + IPageModuleRepository pageModuleRepostory, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageModuleRepostory = pageModuleRepostory; + _pageRepository = pageRepository; + } + + public override string Name => Constants.ModuleSearchIndexManagerName; + + public override int Priority => Constants.ModuleSearchIndexManagerPriority; + + public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + { + var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId); + var searchDocuments = new List(); + + foreach(var pageModule in pageModules) + { + var module = pageModule.Module; + if (module.ModuleDefinition.ServerManagerType != "") + { + _logger.LogDebug($"Search: Begin index module {module.ModuleId}."); + var type = Type.GetType(module.ModuleDefinition.ServerManagerType); + if (type?.GetInterface("IModuleSearch") != null) + { + try + { + var moduleSearch = (IModuleSearch)ActivatorUtilities.CreateInstance(_serviceProvider, type); + var documents = moduleSearch.GetSearchDocuments(module, startTime.GetValueOrDefault(DateTime.MinValue)); + if(documents != null) + { + foreach(var document in documents) + { + SaveModuleMetaData(document, pageModule); + + searchDocuments.Add(document); + } + } + + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index module {module.ModuleId} failed."); + handleError($"Search: Index module {module.ModuleId} failed: {ex.Message}"); + } + } + _logger.LogDebug($"Search: End index module {module.ModuleId}."); + } + } + + processSearchDocuments(searchDocuments); + + return searchDocuments.Count; + } + + private void SaveModuleMetaData(SearchDocument document, PageModule pageModule) + { + + document.EntryId = pageModule.ModuleId; + document.IndexerName = Name; + document.SiteId = pageModule.Module.SiteId; + document.LanguageCode = string.Empty; + + if(document.ModifiedTime == DateTime.MinValue) + { + document.ModifiedTime = pageModule.ModifiedOn; + } + + if (string.IsNullOrEmpty(document.AdditionalContent)) + { + document.AdditionalContent = string.Empty; + } + + var page = _pageRepository.GetPage(pageModule.PageId); + + if (string.IsNullOrEmpty(document.Url) && page != null) + { + document.Url = page.Url; + } + + if (string.IsNullOrEmpty(document.Title) && page != null) + { + document.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name; + } + + if (document.Properties == null) + { + document.Properties = new List(); + } + + if(!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); + } + + if (!document.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); + } + } + } +} diff --git a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs new file mode 100644 index 00000000..55789c23 --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs @@ -0,0 +1,69 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchResultManager : ISearchResultManager + { + public string Name => Constants.ModuleSearchIndexManagerName; + + private readonly IServiceProvider _serviceProvider; + + public ModuleSearchResultManager( + IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)) + { + var page = pageRepository.GetPage(pageId); + if (page != null) + { + return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + } + + return string.Empty; + } + + public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) + { + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + var moduleIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchModuleIdPropertyName)?.Value ?? string.Empty; + if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId) + && !string.IsNullOrEmpty(moduleIdValue) && int.TryParse(moduleIdValue, out int moduleId)) + { + return CanViewPage(pageId, searchQuery.User) && CanViewModule(moduleId, searchQuery.User); + } + + return false; + } + + private bool CanViewModule(int moduleId, User user) + { + var moduleRepository = _serviceProvider.GetRequiredService(); + var module = moduleRepository.GetModule(moduleId); + return module != null && !module.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, module.PermissionList); + } + + private bool CanViewPage(int pageId, User user) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(pageId); + + return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) + && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList)); + } + } +} diff --git a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs new file mode 100644 index 00000000..68c4af02 --- /dev/null +++ b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Mvc.RazorPages; +using System.Reflection.Metadata; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class PageSearchIndexManager : SearchIndexManagerBase + { + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageRepository _pageRepository; + + public PageSearchIndexManager( + IServiceProvider serviceProvider, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageRepository = pageRepository; + } + + public override string Name => Constants.PageSearchIndexManagerName; + + public override int Priority => Constants.PageSearchIndexManagerPriority; + + public override int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError) + { + var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue); + var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue); + var searchDocuments = new List(); + + foreach(var page in pages) + { + try + { + if(IsSystemPage(page)) + { + continue; + } + + var document = new SearchDocument + { + EntryId = page.PageId, + IndexerName = Name, + SiteId = page.SiteId, + LanguageCode = string.Empty, + ModifiedTime = page.ModifiedOn, + AdditionalContent = string.Empty, + Url = page.Url ?? string.Empty, + Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, + Description = string.Empty, + Body = $"{page.Name} {page.Title}" + }; + + if (document.Properties == null) + { + document.Properties = new List(); + } + + if (!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); + } + + searchDocuments.Add(document); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index page {page.PageId} failed."); + handleError($"Search: Index page {page.PageId} failed: {ex.Message}"); + } + } + + processSearchDocuments(searchDocuments); + + return searchDocuments.Count; + } + + private bool IsSystemPage(Models.Page page) + { + return page.Path.Contains("admin") || page.Path == "login" || page.Path == "register" || page.Path == "profile"; + } + } +} diff --git a/Oqtane.Server/Managers/Search/PageSearchResultManager.cs b/Oqtane.Server/Managers/Search/PageSearchResultManager.cs new file mode 100644 index 00000000..fd5456a4 --- /dev/null +++ b/Oqtane.Server/Managers/Search/PageSearchResultManager.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class PageSearchResultManager : ISearchResultManager + { + public string Name => Constants.PageSearchIndexManagerName; + + private readonly IServiceProvider _serviceProvider; + + public PageSearchResultManager( + IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(searchResult.EntryId); + if (page != null) + { + return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + + return string.Empty; + } + + public bool Visible(SearchDocument searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(searchResult.EntryId); + + return page != null && !page.IsDeleted + && UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.View, page.PermissionList) + && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.Edit, page.PermissionList)); + } + } +} diff --git a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs new file mode 100644 index 00000000..0ecb2880 --- /dev/null +++ b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public abstract class SearchIndexManagerBase : ISearchIndexManager + { + private readonly IServiceProvider _serviceProvider; + + public SearchIndexManagerBase(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public abstract int Priority { get; } + + public abstract string Name { get; } + + public abstract int IndexDocuments(int siteId, DateTime? startDate, Action> processSearchDocuments, Action handleError); + + public virtual bool IsIndexEnabled(int siteId) + { + var settingName = string.Format(Constants.SearchIndexManagerEnabledSettingFormat, Name); + var settingRepository = _serviceProvider.GetRequiredService(); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName); + return setting == null || setting.SettingValue == "true"; + } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs new file mode 100644 index 00000000..2f5b6d8d --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs @@ -0,0 +1,64 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentEntityBuilder : AuditableBaseEntityBuilder + { + private const string _entityTableName = "SearchDocument"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocument", x => x.SearchDocumentId); + + public SearchDocumentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + } + + protected override SearchDocumentEntityBuilder BuildTable(ColumnsBuilder table) + { + SearchDocumentId = AddAutoIncrementColumn(table, "SearchDocumentId"); + EntryId = AddIntegerColumn(table, "EntryId"); + IndexerName = AddStringColumn(table, "IndexerName", 50); + SiteId = AddIntegerColumn(table, "SiteId"); + Title = AddStringColumn(table, "Title", 255); + Description = AddMaxStringColumn(table, "Description"); + Body = AddMaxStringColumn(table, "Body"); + Url = AddStringColumn(table, "Url", 255); + ModifiedTime = AddDateTimeColumn(table, "ModifiedTime"); + IsActive = AddBooleanColumn(table, "IsActive"); + AdditionalContent = AddMaxStringColumn(table, "AdditionalContent"); + LanguageCode = AddStringColumn(table, "LanguageCode", 20); + + AddAuditableColumns(table); + + return this; + } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder EntryId { get; private set; } + + public OperationBuilder IndexerName { get; private set; } + + public OperationBuilder SiteId { get; private set; } + + public OperationBuilder Title { get; private set; } + + public OperationBuilder Description { get; private set; } + + public OperationBuilder Body { get; private set; } + + public OperationBuilder Url { get; private set; } + + public OperationBuilder ModifiedTime { get; private set; } + + public OperationBuilder IsActive { get; private set; } + + public OperationBuilder AdditionalContent { get; private set; } + + public OperationBuilder LanguageCode { get; private set; } + + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs new file mode 100644 index 00000000..a591aec2 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentPropertyEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchDocumentProperty"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentProperty", x => x.PropertyId); + private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentProperty_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); + + public SearchDocumentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchDocumentForeignKey); + } + + protected override SearchDocumentPropertyEntityBuilder BuildTable(ColumnsBuilder table) + { + PropertyId = AddAutoIncrementColumn(table, "PropertyId"); + SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); + Name = AddStringColumn(table, "Name", 50); + Value = AddStringColumn(table, "Value", 50); + + return this; + } + + public OperationBuilder PropertyId { get; private set; } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder Name { get; private set; } + + public OperationBuilder Value { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs new file mode 100644 index 00000000..bd864f22 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs @@ -0,0 +1,37 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchDocumentTagEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchDocumentTag"; + private readonly PrimaryKey _primaryKey = new("PK_SearchDocumentTag", x => x.TagId); + private readonly ForeignKey _searchDocumentForeignKey = new("FK_SearchDocumentTag_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade); + + public SearchDocumentTagEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchDocumentForeignKey); + } + + protected override SearchDocumentTagEntityBuilder BuildTable(ColumnsBuilder table) + { + TagId = AddAutoIncrementColumn(table, "TagId"); + SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId"); + Tag = AddStringColumn(table, "Tag", 50); + + return this; + } + + public OperationBuilder TagId { get; private set; } + + public OperationBuilder SearchDocumentId { get; private set; } + + public OperationBuilder Tag { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs new file mode 100644 index 00000000..f32a8137 --- /dev/null +++ b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Oqtane.Databases.Interfaces; +using Oqtane.Migrations.EntityBuilders; +using Oqtane.Repository; + +namespace Oqtane.Migrations.Tenant +{ + [DbContext(typeof(TenantDBContext))] + [Migration("Tenant.05.02.00.01")] + public class AddSearchTables : MultiDatabaseMigration + { + public AddSearchTables(IDatabase database) : base(database) + { + } + + protected override void Up(MigrationBuilder migrationBuilder) + { + var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentEntityBuilder.Create(); + + var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentPropertyEntityBuilder.Create(); + + var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentTagEntityBuilder.Create(); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentPropertyEntityBuilder.Drop(); + + var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentTagEntityBuilder.Drop(); + + var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase); + searchDocumentEntityBuilder.Drop(); + } + } +} diff --git a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs index 6c9293fb..8cabf391 100644 --- a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs +++ b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs @@ -8,20 +8,29 @@ using Oqtane.Shared; using Oqtane.Migrations.Framework; using Oqtane.Documentation; using System.Linq; +using Oqtane.Interfaces; +using System.Collections.Generic; +using System; // ReSharper disable ConvertToUsingDeclaration namespace Oqtane.Modules.HtmlText.Manager { [PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")] - public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable + public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, IModuleSearch { + private readonly IServiceProvider _serviceProvider; private readonly IHtmlTextRepository _htmlText; private readonly IDBContextDependencies _DBContextDependencies; private readonly ISqlRepository _sqlRepository; - public HtmlTextManager(IHtmlTextRepository htmlText, IDBContextDependencies DBContextDependencies, ISqlRepository sqlRepository) + public HtmlTextManager( + IServiceProvider serviceProvider, + IHtmlTextRepository htmlText, + IDBContextDependencies DBContextDependencies, + ISqlRepository sqlRepository) { + _serviceProvider = serviceProvider; _htmlText = htmlText; _DBContextDependencies = DBContextDependencies; _sqlRepository = sqlRepository; @@ -39,6 +48,27 @@ namespace Oqtane.Modules.HtmlText.Manager return content; } + public IList GetSearchDocuments(Module module, DateTime startDate) + { + var searchDocuments = new List(); + + var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId); + if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate)) + { + var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First(); + + searchDocuments.Add(new SearchDocument + { + Title = module.Title, + Description = string.Empty, + Body = SearchUtils.Clean(htmltext.Content, true), + ModifiedTime = htmltext.ModifiedOn + }); + } + + return searchDocuments; + } + public void ImportModule(Module module, string content, string version) { content = WebUtility.HtmlDecode(content); diff --git a/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs b/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs new file mode 100644 index 00000000..d134b1ae --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs @@ -0,0 +1,43 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Oqtane.Controllers; +using Oqtane.Documentation; +using Oqtane.Enums; +using Oqtane.Infrastructure; +using Oqtane.Modules.SearchResults.Services; +using Oqtane.Shared; + +namespace Oqtane.Modules.SearchResults.Controllers +{ + [Route(ControllerRoutes.ApiRoute)] + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class SearchResultsController : ModuleControllerBase + { + private readonly ISearchResultsService _searchResultsService; + + public SearchResultsController(ISearchResultsService searchResultsService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor) + { + _searchResultsService = searchResultsService; + } + + [HttpPost] + [Authorize(Policy = PolicyNames.ViewModule)] + public async Task Post([FromBody] Models.SearchQuery searchQuery) + { + try + { + return await _searchResultsService.SearchAsync(AuthEntityId(EntityNames.Module), searchQuery); + } + catch(Exception ex) + { + _logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery); + HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return null; + } + } + } +} diff --git a/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs b/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs new file mode 100644 index 00000000..3823bac6 --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs @@ -0,0 +1,36 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Oqtane.Documentation; +using Oqtane.Infrastructure; +using Oqtane.Models; +using Oqtane.Services; + +namespace Oqtane.Modules.SearchResults.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public class ServerSearchResultsService : ISearchResultsService, ITransientService + { + private readonly ILogManager _logger; + private readonly IHttpContextAccessor _accessor; + private readonly Alias _alias; + private readonly ISearchService _searchService; + + public ServerSearchResultsService( + ITenantManager tenantManager, + ILogManager logger, + IHttpContextAccessor accessor, + ISearchService searchService) + { + _logger = logger; + _accessor = accessor; + _alias = tenantManager.GetAlias(); + _searchService = searchService; + } + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + var results = await _searchService.SearchAsync(searchQuery); + return results; + } + } +} diff --git a/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs b/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs new file mode 100644 index 00000000..25d2d15f --- /dev/null +++ b/Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Infrastructure; +using Oqtane.Modules.SearchResults.Services; + +namespace Oqtane.Modules.SearchResults.Startup +{ + public class ServerStartup : IServerStartup + { + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + } + + public void ConfigureMvc(IMvcBuilder mvcBuilder) + { + } + + public void ConfigureServices(IServiceCollection services) + { + services.AddTransient(); + } + } +} diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs new file mode 100644 index 00000000..ac2ed576 --- /dev/null +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; +using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; + +namespace Oqtane.Providers +{ + public class DatabaseSearchProvider : ISearchProvider + { + private readonly ISearchDocumentRepository _searchDocumentRepository; + + private const float TitleBoost = 100f; + private const float DescriptionBoost = 10f; + private const float BodyBoost = 10f; + private const float AdditionalContentBoost = 5f; + + public string Name => Constants.DefaultSearchProviderName; + + public DatabaseSearchProvider(ISearchDocumentRepository searchDocumentRepository) + { + _searchDocumentRepository = searchDocumentRepository; + } + + public void Commit() + { + } + + public void DeleteDocument(string id) + { + _searchDocumentRepository.DeleteSearchDocument(id); + } + + public bool Optimize() + { + return true; + } + + public void ResetIndex() + { + _searchDocumentRepository.DeleteAllSearchDocuments(); + } + + public void SaveDocument(SearchDocument document, bool autoCommit = false) + { + //remove exist document + _searchDocumentRepository.DeleteSearchDocument(document.IndexerName, document.EntryId); + + _searchDocumentRepository.AddSearchDocument(document); + } + + public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) + { + var totalResults = 0; + + var documents = await _searchDocumentRepository.GetSearchDocumentsAsync(searchQuery); + + //convert the search documents to search results. + var results = documents + .Where(i => validateFunc(i, searchQuery)) + .Select(i => ConvertToSearchResult(i, searchQuery)); + + if (searchQuery.SortDirection == SearchSortDirections.Descending) + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderByDescending(i => i.ModifiedTime); + break; + } + } + else + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderBy(i => i.ModifiedTime); + break; + } + } + + //remove duplicated results based on page id for Page and Module types + results = results.DistinctBy(i => + { + if (i.IndexerName == Constants.PageSearchIndexManagerName || i.IndexerName == Constants.ModuleSearchIndexManagerName) + { + var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey; + } + else + { + return i.UniqueKey; + } + }); + + totalResults = results.Count(); + + return new SearchResults + { + Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(), + TotalResults = totalResults + }; + } + + private SearchResult ConvertToSearchResult(SearchDocument searchDocument, SearchQuery searchQuery) + { + var searchResult = new SearchResult() + { + SearchDocumentId = searchDocument.SearchDocumentId, + SiteId = searchDocument.SiteId, + IndexerName = searchDocument.IndexerName, + EntryId = searchDocument.EntryId, + Title = searchDocument.Title, + Description = searchDocument.Description, + Body = searchDocument.Body, + Url = searchDocument.Url, + ModifiedTime = searchDocument.ModifiedTime, + Tags = searchDocument.Tags, + Properties = searchDocument.Properties, + Snippet = BuildSnippet(searchDocument, searchQuery), + Score = CalculateScore(searchDocument, searchQuery) + }; + + return searchResult; + } + + private float CalculateScore(SearchDocument searchDocument, SearchQuery searchQuery) + { + var score = 0f; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + score += Regex.Matches(searchDocument.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost; + score += Regex.Matches(searchDocument.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost; + score += Regex.Matches(searchDocument.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost; + score += Regex.Matches(searchDocument.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost; + } + + return score / 100; + } + + private string BuildSnippet(SearchDocument searchDocument, SearchQuery searchQuery) + { + var content = $"{searchDocument.Title} {searchDocument.Description} {searchDocument.Body}"; + var snippet = string.Empty; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + if (!string.IsNullOrWhiteSpace(keyword) && content.Contains(keyword, StringComparison.OrdinalIgnoreCase)) + { + var start = content.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) - 20; + var prefix = "..."; + var suffix = "..."; + if (start <= 0) + { + start = 0; + prefix = string.Empty; + } + + var length = searchQuery.BodySnippetLength; + if (start + length >= content.Length) + { + length = content.Length - start; + suffix = string.Empty; + } + + snippet = $"{prefix}{content.Substring(start, length)}{suffix}"; + break; + + + } + } + + if (string.IsNullOrEmpty(snippet)) + { + snippet = content.Substring(0, searchQuery.BodySnippetLength); + } + + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + snippet = Regex.Replace(snippet, $"({keyword})", $"$1", RegexOptions.IgnoreCase); + } + + return snippet; + } + } +} diff --git a/Oqtane.Server/Repository/Context/TenantDBContext.cs b/Oqtane.Server/Repository/Context/TenantDBContext.cs index 2efad3f7..cbe57631 100644 --- a/Oqtane.Server/Repository/Context/TenantDBContext.cs +++ b/Oqtane.Server/Repository/Context/TenantDBContext.cs @@ -29,5 +29,8 @@ namespace Oqtane.Repository public virtual DbSet Language { get; set; } public virtual DbSet Visitor { get; set; } public virtual DbSet UrlMapping { get; set; } + public virtual DbSet SearchDocument { get; set; } + public virtual DbSet SearchDocumentProperty { get; set; } + public virtual DbSet SearchDocumentTag { get; set; } } } diff --git a/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs b/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs new file mode 100644 index 00000000..6128c40c --- /dev/null +++ b/Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Repository +{ + public interface ISearchDocumentRepository + { + Task> GetSearchDocumentsAsync(SearchQuery searchQuery); + SearchDocument AddSearchDocument(SearchDocument searchDocument); + void DeleteSearchDocument(int searchDocumentId); + void DeleteSearchDocument(string indexerName, int entryId); + void DeleteSearchDocument(string uniqueKey); + void DeleteAllSearchDocuments(); + } +} diff --git a/Oqtane.Server/Repository/SearchDocumentRepository.cs b/Oqtane.Server/Repository/SearchDocumentRepository.cs new file mode 100644 index 00000000..bdceb52e --- /dev/null +++ b/Oqtane.Server/Repository/SearchDocumentRepository.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Repository +{ + public class SearchDocumentRepository : ISearchDocumentRepository + { + private readonly IDbContextFactory _dbContextFactory; + + public SearchDocumentRepository(IDbContextFactory dbContextFactory) + { + _dbContextFactory = dbContextFactory; + } + + public async Task> GetSearchDocumentsAsync(SearchQuery searchQuery) + { + using var db = _dbContextFactory.CreateDbContext(); + var documents = db.SearchDocument.AsNoTracking() + .Include(i => i.Properties) + .Include(i => i.Tags) + .Where(i => i.SiteId == searchQuery.SiteId); + + if (searchQuery.Sources != null && searchQuery.Sources.Any()) + { + documents = documents.Where(i => searchQuery.Sources.Contains(i.IndexerName)); + } + + if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue) + { + documents = documents.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc); + } + + if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue) + { + documents = documents.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc); + } + + if (searchQuery.Tags != null && searchQuery.Tags.Any()) + { + foreach (var tag in searchQuery.Tags) + { + documents = documents.Where(i => i.Tags.Any(t => t.Tag == tag)); + } + } + + if (searchQuery.Properties != null && searchQuery.Properties.Any()) + { + foreach (var property in searchQuery.Properties) + { + documents = documents.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value)); + } + } + + var filteredDocuments = new List(); + if (!string.IsNullOrEmpty(searchQuery.Keywords)) + { + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + filteredDocuments.AddRange(await documents.Where(i => i.Title.Contains(keyword) || i.Description.Contains(keyword) || i.Body.Contains(keyword)).ToListAsync()); + } + } + + return filteredDocuments.DistinctBy(i => i.UniqueKey); + } + + public SearchDocument AddSearchDocument(SearchDocument searchDocument) + { + using var context = _dbContextFactory.CreateDbContext(); + context.SearchDocument.Add(searchDocument); + + if(searchDocument.Properties != null && searchDocument.Properties.Any()) + { + foreach(var property in searchDocument.Properties) + { + property.SearchDocumentId = searchDocument.SearchDocumentId; + context.SearchDocumentProperty.Add(property); + } + } + + if (searchDocument.Tags != null && searchDocument.Tags.Any()) + { + foreach (var tag in searchDocument.Tags) + { + tag.SearchDocumentId = searchDocument.SearchDocumentId; + context.SearchDocumentTag.Add(tag); + } + } + + context.SaveChanges(); + + return searchDocument; + } + + public void DeleteSearchDocument(int searchDocumentId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.Find(searchDocumentId); + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + + public void DeleteSearchDocument(string indexerName, int entryId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.FirstOrDefault(i => i.IndexerName == indexerName && i.EntryId == entryId); + if(searchDocument != null) + { + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + } + + public void DeleteSearchDocument(string uniqueKey) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchDocument = db.SearchDocument.FirstOrDefault(i => (i.IndexerName + ":" + i.EntryId) == uniqueKey); + if (searchDocument != null) + { + db.SearchDocument.Remove(searchDocument); + db.SaveChanges(); + } + } + + public void DeleteAllSearchDocuments() + { + using var db = _dbContextFactory.CreateDbContext(); + db.SearchDocument.RemoveRange(db.SearchDocument); + db.SaveChanges(); + } + } +} diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs new file mode 100644 index 00000000..caa9fce6 --- /dev/null +++ b/Oqtane.Server/Services/SearchService.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Infrastructure; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + public class SearchService : ISearchService + { + private readonly IServiceProvider _serviceProvider; + private readonly ITenantManager _tenantManager; + private readonly IAliasRepository _aliasRepository; + private readonly ISettingRepository _settingRepository; + private readonly ILogger _logger; + private readonly IMemoryCache _cache; + + public SearchService( + IServiceProvider serviceProvider, + ITenantManager tenantManager, + IAliasRepository aliasRepository, + ISettingRepository settingRepository, + ILogger logger, + IMemoryCache cache) + { + _tenantManager = tenantManager; + _aliasRepository = aliasRepository; + _settingRepository = settingRepository; + _serviceProvider = serviceProvider; + _logger = logger; + _cache = cache; + } + + public void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError) + { + var searchEnabled = SearchEnabled(siteId); + if(!searchEnabled) + { + logNote($"Search: Search is disabled on site {siteId}.
"); + return; + } + + _logger.LogDebug($"Search: Start Index Content of {siteId}, Start Time: {startTime.GetValueOrDefault(DateTime.MinValue)}"); + + var searchProvider = GetSearchProvider(siteId); + + SetTenant(siteId); + + if (startTime == null) + { + searchProvider.ResetIndex(); + } + + var searchIndexManagers = GetSearchIndexManagers(m => { }); + foreach (var searchIndexManager in searchIndexManagers) + { + if (!searchIndexManager.IsIndexEnabled(siteId)) + { + logNote($"Search: Ignore indexer {searchIndexManager.Name} because it's disabled.
"); + } + else + { + _logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}"); + + var count = searchIndexManager.IndexDocuments(siteId, startTime, SaveIndexDocuments, handleError); + logNote($"Search: Indexer {searchIndexManager.Name} processed {count} documents.
"); + + _logger.LogDebug($"Search: End Index {searchIndexManager.Name}"); + } + } + } + + public async Task SearchAsync(SearchQuery searchQuery) + { + var searchProvider = GetSearchProvider(searchQuery.SiteId); + var searchResults = await searchProvider.SearchAsync(searchQuery, HasViewPermission); + + //generate the document url if it's not set. + foreach (var result in searchResults.Results) + { + if(string.IsNullOrEmpty(result.Url)) + { + result.Url = GetDocumentUrl(result, searchQuery); + } + } + + return searchResults; + } + + private ISearchProvider GetSearchProvider(int siteId) + { + var providerName = GetSearchProviderSetting(siteId); + var searchProviders = _serviceProvider.GetServices(); + var provider = searchProviders.FirstOrDefault(i => i.Name == providerName); + if(provider == null) + { + provider = searchProviders.FirstOrDefault(i => i.Name == Constants.DefaultSearchProviderName); + } + + return provider; + } + + private string GetSearchProviderSetting(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchProviderSettingName); + if(!string.IsNullOrEmpty(setting?.SettingValue)) + { + return setting.SettingValue; + } + + return Constants.DefaultSearchProviderName; + } + + private bool SearchEnabled(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchEnabledSettingName); + if (!string.IsNullOrEmpty(setting?.SettingValue)) + { + return bool.TryParse(setting.SettingValue, out bool enabled) && enabled; + } + + return true; + } + + private void SetTenant(int siteId) + { + var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId); + _tenantManager.SetAlias(alias); + } + + private IList GetSearchIndexManagers(Action initManager) + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchIndexManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchIndexManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + initManager(manager); + managers.Add(manager); + } + + return managers.OrderBy(i => i.Priority).ToList(); + } + + private IList GetSearchResultManagers() + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + managers.Add(manager); + } + + return managers.ToList(); + } + + private void SaveIndexDocuments(IList searchDocuments) + { + if(searchDocuments.Any()) + { + var searchProvider = GetSearchProvider(searchDocuments.First().SiteId); + + foreach (var searchDocument in searchDocuments) + { + try + { + searchProvider.SaveDocument(searchDocument); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Save search document {searchDocument.UniqueKey} failed."); + } + } + + //commit the index changes + searchProvider.Commit(); + } + } + + private bool HasViewPermission(SearchDocument searchDocument, SearchQuery searchQuery) + { + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchDocument.IndexerName); + if (searchResultManager != null) + { + return searchResultManager.Visible(searchDocument, searchQuery); + } + return true; + } + + private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery) + { + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.IndexerName); + if(searchResultManager != null) + { + return searchResultManager.GetUrl(result, searchQuery); + } + + return string.Empty; + } + } +} diff --git a/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css new file mode 100644 index 00000000..71194e38 --- /dev/null +++ b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css @@ -0,0 +1,3 @@ +.search-result-container ul.pagination li label, .search-result-container ul.dropdown-menu li label { + cursor: pointer; +} \ No newline at end of file diff --git a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css index 1b9ea409..bbb65908 100644 --- a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css +++ b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css @@ -79,6 +79,10 @@ body { top: -2px; } +.app-search input{ + width: auto; +} + .navbar-toggler { background-color: rgba(255, 255, 255, 0.1); margin: .5rem; diff --git a/Oqtane.Server/wwwroot/css/app.css b/Oqtane.Server/wwwroot/css/app.css index f5a65a80..941ff5f1 100644 --- a/Oqtane.Server/wwwroot/css/app.css +++ b/Oqtane.Server/wwwroot/css/app.css @@ -235,3 +235,17 @@ app { .app-form-inline { display: inline-block; } +.app-search{ + display: inline-block; + position: relative; +} +.app-search input + button{ + background: none; + border: none; + position: absolute; + right: 0; + top: 0; +} +.app-search input + button .oi{ + top: 0; +} \ No newline at end of file diff --git a/Oqtane.Shared/Enums/SearchSortDirections.cs b/Oqtane.Shared/Enums/SearchSortDirections.cs new file mode 100644 index 00000000..62c7f08d --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortDirections.cs @@ -0,0 +1,10 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortDirections + { + Ascending, + Descending + } +} diff --git a/Oqtane.Shared/Enums/SearchSortFields.cs b/Oqtane.Shared/Enums/SearchSortFields.cs new file mode 100644 index 00000000..c8e16175 --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortFields.cs @@ -0,0 +1,11 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortFields + { + Relevance, + Title, + LastModified + } +} diff --git a/Oqtane.Shared/Interfaces/IModuleSearch.cs b/Oqtane.Shared/Interfaces/IModuleSearch.cs new file mode 100644 index 00000000..af94a300 --- /dev/null +++ b/Oqtane.Shared/Interfaces/IModuleSearch.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Interfaces +{ + public interface IModuleSearch + { + public IList GetSearchDocuments(Module module, DateTime startTime); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs new file mode 100644 index 00000000..9a322ab5 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Web; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchIndexManager + { + int Priority { get; } + + string Name { get; } + + bool IsIndexEnabled(int siteId); + + int IndexDocuments(int siteId, DateTime? startTime, Action> processSearchDocuments, Action handleError); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchProvider.cs b/Oqtane.Shared/Interfaces/ISearchProvider.cs new file mode 100644 index 00000000..e1e9fb16 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchProvider.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchProvider + { + string Name { get; } + + void SaveDocument(SearchDocument document, bool autoCommit = false); + + void DeleteDocument(string id); + + Task SearchAsync(SearchQuery searchQuery, Func validateFunc); + + bool Optimize(); + + void Commit(); + + void ResetIndex(); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchResultManager.cs b/Oqtane.Shared/Interfaces/ISearchResultManager.cs new file mode 100644 index 00000000..2dc459b0 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchResultManager.cs @@ -0,0 +1,14 @@ +using System; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchResultManager + { + string Name { get; } + + bool Visible(SearchDocument searchResult, SearchQuery searchQuery); + + string GetUrl(SearchResult searchResult, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchService.cs b/Oqtane.Shared/Interfaces/ISearchService.cs new file mode 100644 index 00000000..0f125c65 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchService.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchService + { + void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError); + + Task SearchAsync(SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Models/SearchDocument.cs b/Oqtane.Shared/Models/SearchDocument.cs new file mode 100644 index 00000000..67161b3b --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocument.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Text.Json; +namespace Oqtane.Models +{ + public class SearchDocument : ModelBase + { + public int SearchDocumentId { get; set; } + + [NotMapped] + public string UniqueKey => $"{IndexerName}:{EntryId}"; + + public int EntryId { get; set; } + + public string IndexerName { get; set; } + + public int SiteId { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Body { get; set; } + + public string Url { get; set; } + + public DateTime ModifiedTime { get; set; } + + public bool IsActive { get; set; } + + public string AdditionalContent { get; set; } + + public string LanguageCode { get; set; } + + public IList Tags { get; set; } + + public IList Properties { get; set; } + + public override string ToString() + { + return JsonSerializer.Serialize(this); + } + } +} diff --git a/Oqtane.Shared/Models/SearchDocumentProperty.cs b/Oqtane.Shared/Models/SearchDocumentProperty.cs new file mode 100644 index 00000000..4febae43 --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocumentProperty.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.EntityFrameworkCore; + +namespace Oqtane.Models +{ + public class SearchDocumentProperty + { + [Key] + public int PropertyId { get; set; } + + public int SearchDocumentId { get; set; } + + public string Name { get; set; } + + public string Value { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchDocumentTag.cs b/Oqtane.Shared/Models/SearchDocumentTag.cs new file mode 100644 index 00000000..ee929efc --- /dev/null +++ b/Oqtane.Shared/Models/SearchDocumentTag.cs @@ -0,0 +1,14 @@ +using System.ComponentModel.DataAnnotations; + +namespace Oqtane.Models +{ + public class SearchDocumentTag + { + [Key] + public int TagId { get; set; } + + public int SearchDocumentId { get; set; } + + public string Tag { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchQuery.cs b/Oqtane.Shared/Models/SearchQuery.cs new file mode 100644 index 00000000..f148252d --- /dev/null +++ b/Oqtane.Shared/Models/SearchQuery.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using Oqtane.Shared; + +namespace Oqtane.Models +{ + public class SearchQuery + { + public int SiteId { get; set; } + + public Alias Alias { get; set; } + + public User User { get; set; } + + public string Keywords { get; set; } + + public IList Sources { get; set; } = new List(); + + public DateTime BeginModifiedTimeUtc { get; set; } + + public DateTime EndModifiedTimeUtc { get; set; } + + public IList Tags { get; set; } = new List(); + + public IDictionary Properties { get; set; } = new Dictionary(); + + public int PageIndex { get; set; } + + public int PageSize { get; set; } + + public SearchSortFields SortField { get; set; } + + public SearchSortDirections SortDirection { get; set; } + + public int BodySnippetLength { get; set;} = 255; + } +} diff --git a/Oqtane.Shared/Models/SearchResult.cs b/Oqtane.Shared/Models/SearchResult.cs new file mode 100644 index 00000000..d13cafa1 --- /dev/null +++ b/Oqtane.Shared/Models/SearchResult.cs @@ -0,0 +1,11 @@ +namespace Oqtane.Models +{ + public class SearchResult : SearchDocument + { + public float Score { get; set; } + + public string DisplayScore { get; set; } + + public string Snippet { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchResults.cs b/Oqtane.Shared/Models/SearchResults.cs new file mode 100644 index 00000000..5642468c --- /dev/null +++ b/Oqtane.Shared/Models/SearchResults.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Oqtane.Models +{ + public class SearchResults + { + public IList Results { get; set; } + + public int TotalResults { get; set; } + } +} diff --git a/Oqtane.Shared/Shared/Constants.cs b/Oqtane.Shared/Shared/Constants.cs index b2b8c837..f1a14cd4 100644 --- a/Oqtane.Shared/Shared/Constants.cs +++ b/Oqtane.Shared/Shared/Constants.cs @@ -77,6 +77,22 @@ namespace Oqtane.Shared public static readonly string VisitorCookiePrefix = "APP_VISITOR_"; + public const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; + public const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; + public const string SearchResultManagersCacheName = "SearchResultManagers"; + public const int SearchDefaultPageSize = 10; + public const string SearchPageIdPropertyName = "PageId"; + public const string SearchModuleIdPropertyName = "ModuleId"; + public const string DefaultSearchProviderName = "Database"; + public const string SearchProviderSettingName = "SearchProvider"; + public const string SearchEnabledSettingName = "SearchEnabled"; + + public const string ModuleSearchIndexManagerName = "Module"; + public const string PageSearchIndexManagerName = "Page"; + + public const int PageSearchIndexManagerPriority = 100; + public const int ModuleSearchIndexManagerPriority = 200; + // Obsolete constants const string RoleObsoleteMessage = "Use the corresponding member from Oqtane.Shared.RoleNames"; diff --git a/Oqtane.Shared/Shared/SearchUtils.cs b/Oqtane.Shared/Shared/SearchUtils.cs new file mode 100644 index 00000000..b7116d4a --- /dev/null +++ b/Oqtane.Shared/Shared/SearchUtils.cs @@ -0,0 +1,95 @@ +using System.Collections; +using System.Collections.Generic; +using System.Net; +using System.Text.RegularExpressions; + +namespace Oqtane.Shared +{ + public sealed class SearchUtils + { + private const string PunctuationMatch = "[~!#\\$%\\^&*\\(\\)-+=\\{\\[\\}\\]\\|;:\\x22'<,>\\.\\?\\\\\\t\\r\\v\\f\\n]"; + private static readonly Regex _stripWhiteSpaceRegex = new Regex("\\s+", RegexOptions.Compiled); + private static readonly Regex _stripTagsRegex = new Regex("<[^<>]*>", RegexOptions.Compiled); + private static readonly Regex _afterRegEx = new Regex(PunctuationMatch + "\\s", RegexOptions.Compiled); + private static readonly Regex _beforeRegEx = new Regex("\\s" + PunctuationMatch, RegexOptions.Compiled); + + public static string Clean(string html, bool removePunctuation) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + if (html.Contains("<")) + { + html = WebUtility.HtmlDecode(html); + } + + html = StripTags(html, true); + html = WebUtility.HtmlDecode(html); + + if (removePunctuation) + { + html = StripPunctuation(html, true); + html = StripWhiteSpace(html, true); + } + + return html; + } + + public static IList GetKeywordsList(string keywords) + { + var keywordsList = new List(); + if(!string.IsNullOrEmpty(keywords)) + { + foreach (var keyword in keywords.Split(' ')) + { + if (!string.IsNullOrWhiteSpace(keyword.Trim())) + { + keywordsList.Add(keyword.Trim()); + } + } + } + + return keywordsList; + } + + private static string StripTags(string html, bool retainSpace) + { + return _stripTagsRegex.Replace(html, retainSpace ? " " : string.Empty); + } + + private static string StripPunctuation(string html, bool retainSpace) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + string retHTML = html + " "; + + var repString = retainSpace ? " " : string.Empty; + while (_beforeRegEx.IsMatch(retHTML)) + { + retHTML = _beforeRegEx.Replace(retHTML, repString); + } + + while (_afterRegEx.IsMatch(retHTML)) + { + retHTML = _afterRegEx.Replace(retHTML, repString); + } + + return retHTML.Trim('"'); + } + + private static string StripWhiteSpace(string html, bool retainSpace) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + return _stripWhiteSpaceRegex.Replace(html, retainSpace ? " " : string.Empty); + } + } +}