diff --git a/Oqtane.Client/Modules/Admin/SearchResults/Index.razor b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor new file mode 100644 index 00000000..3b25090e --- /dev/null +++ b/Oqtane.Client/Modules/Admin/SearchResults/Index.razor @@ -0,0 +1,135 @@ +@using Microsoft.AspNetCore.Http +@using Oqtane.Services +@using System.Net +@namespace Oqtane.Modules.Admin.SearchResults +@inherits ModuleBase +@inject ISearchResultsService SearchResultsService +@inject IStringLocalizer Localizer +@inject IHttpContextAccessor HttpContext + +
+
+
+
+
+ @Localizer["SearchPrefix"] + + + +
+
+
+
+
+
+ @if (_loading) + { +
+ } + else + { + @if (_searchResults != null && _searchResults.Results != null) + { + if (_searchResults.Results.Any()) + { + + +
+

@context.Title

+

@((MarkupString)context.Snippet)

+
+ + + } + else + { + + } + } +
+ } +
+
+
+@code { + public override string RenderMode => RenderModes.Static; + private const int SearchDefaultPageSize = 10; + + private SearchSortDirections _searchSortDirection = SearchSortDirections.Descending; //default sort by + private SearchSortFields _searchSortField = SearchSortFields.Relevance; + private string _keywords; + private bool _loading; + private SearchResults _searchResults; + private int _currentPage = 0; + private int _pageSize = SearchDefaultPageSize; + private int _displayPages = 7; + + protected override async Task OnInitializedAsync() + { + if (ModuleState.Settings.ContainsKey("PageSize")) + { + _pageSize = int.Parse(ModuleState.Settings["PageSize"]); + } + + if (PageState.QueryString.ContainsKey("q")) + { + _keywords = WebUtility.UrlDecode(PageState.QueryString["q"]); + } + + if (!string.IsNullOrEmpty(_keywords)) + { + await PerformSearch(); + } + } + + private async Task Search() + { + _keywords = HttpContext.HttpContext.Request.Form["keywords"]; + if (string.IsNullOrEmpty(_keywords)) + { + AddModuleMessage(Localizer["MissingKeywords"], MessageType.Warning); + } + else + { + ClearModuleMessage(); + + _currentPage = 0; + await PerformSearch(); + } + } + + private async Task PerformSearch() + { + _loading = true; + StateHasChanged(); + + var searchQuery = new SearchQuery + { + SiteId = PageState.Site.SiteId, + Alias = PageState.Alias, + User = PageState.User, + Keywords = _keywords, + SortDirection = _searchSortDirection, + SortField = _searchSortField, + PageIndex = 0, + PageSize = int.MaxValue + }; + + _searchResults = await SearchResultsService.SearchAsync(ModuleState.ModuleId, searchQuery); + + _loading = false; + StateHasChanged(); + } +} \ No newline at end of file diff --git a/Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs b/Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs new file mode 100644 index 00000000..97a6835e --- /dev/null +++ b/Oqtane.Client/Modules/Admin/SearchResults/ModuleInfo.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Modules.Admin.SearchResults +{ + [PrivateApi("Mark this as private, since it's not very useful in the public docs")] + public class ModuleInfo : IModule + { + public ModuleDefinition ModuleDefinition => new ModuleDefinition + { + Name = "Search Results", + Description = "Display Search Results", + Version = Constants.Version, + ServerManagerType = "", + SettingsType = "Oqtane.Modules.Admin.SearchResults.Settings, Oqtane.Client", + Resources = new List() + { + new Resource { ResourceType = ResourceType.Stylesheet, Url = "~/Module.css" } + } + }; + } +} diff --git a/Oqtane.Client/Modules/Admin/SearchResults/Settings.razor b/Oqtane.Client/Modules/Admin/SearchResults/Settings.razor new file mode 100644 index 00000000..1e98f0f6 --- /dev/null +++ b/Oqtane.Client/Modules/Admin/SearchResults/Settings.razor @@ -0,0 +1,48 @@ +@namespace Oqtane.Modules.Admin.SearchResults +@inherits ModuleBase +@implements Oqtane.Interfaces.ISettingsControl +@inject ISettingService SettingService +@inject IStringLocalizer Localizer +@inject IStringLocalizer SharedLocalizer + +
+
+ +
+ +
+
+
+ +@code { + private const string SearchDefaultPageSize = "10"; + + private string resourceType = "Oqtane.Modules.Admin.SearchResults.Settings, Oqtane.Client"; // for localization + private string _pageSize; + + protected override void OnInitialized() + { + try + { + _pageSize = SettingService.GetSetting(ModuleState.Settings, "PageSize", SearchDefaultPageSize); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } + + public async Task UpdateSettings() + { + try + { + var settings = await SettingService.GetModuleSettingsAsync(ModuleState.ModuleId); + settings = SettingService.SetSetting(settings, "PageSize", _pageSize); + await SettingService.UpdateModuleSettingsAsync(settings, ModuleState.ModuleId); + } + catch (Exception ex) + { + AddModuleMessage(ex.Message, MessageType.Error); + } + } +} diff --git a/Oqtane.Client/Resources/Modules/Admin/SearchResults/Index.resx b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Index.resx new file mode 100644 index 00000000..bd07f3f8 --- /dev/null +++ b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Index.resx @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Ascending + + + Descending + + + Please provide the search keywords. + + + Modification Time + + + No results found. + + + Relevance + + + Search + + + Search + + + Search: + + + Sort Direction + + + Sort By + + + Title + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Modules/Admin/SearchResults/Settings.resx b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Settings.resx new file mode 100644 index 00000000..1c48707d --- /dev/null +++ b/Oqtane.Client/Resources/Modules/Admin/SearchResults/Settings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Page Size + + \ No newline at end of file diff --git a/Oqtane.Client/Resources/Themes/Controls/Search.resx b/Oqtane.Client/Resources/Themes/Controls/Search.resx new file mode 100644 index 00000000..c2d2b4ee --- /dev/null +++ b/Oqtane.Client/Resources/Themes/Controls/Search.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Search + + + Search + + \ No newline at end of file diff --git a/Oqtane.Client/Services/Interfaces/ISearchResultsService.cs b/Oqtane.Client/Services/Interfaces/ISearchResultsService.cs new file mode 100644 index 00000000..0671117c --- /dev/null +++ b/Oqtane.Client/Services/Interfaces/ISearchResultsService.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; + +namespace Oqtane.Services +{ + [PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")] + public interface ISearchResultsService + { + Task SearchAsync(int moduleId, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Client/Services/SearchResultsService.cs b/Oqtane.Client/Services/SearchResultsService.cs new file mode 100644 index 00000000..da9687d5 --- /dev/null +++ b/Oqtane.Client/Services/SearchResultsService.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Oqtane.Documentation; +using Oqtane.Models; +using Oqtane.Modules; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + [PrivateApi("Don't show in the documentation, as everything should use the Interface")] + public class SearchResultsService : ServiceBase, ISearchResultsService, IClientService + { + public SearchResultsService(HttpClient http, SiteState siteState) : base(http, siteState) { } + + private string ApiUrl => CreateApiUrl("SearchResults"); + + public async Task SearchAsync(int moduleId, SearchQuery searchQuery) + { + return await PostJsonAsync(CreateAuthorizationPolicyUrl(ApiUrl, EntityNames.Module, moduleId), searchQuery); + } + } +} diff --git a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor index 553d97d1..2e0da4d3 100644 --- a/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/BlazorTheme/Themes/Default.razor @@ -9,7 +9,9 @@
diff --git a/Oqtane.Client/Themes/Controls/Theme/Search.razor b/Oqtane.Client/Themes/Controls/Theme/Search.razor new file mode 100644 index 00000000..a782bdaf --- /dev/null +++ b/Oqtane.Client/Themes/Controls/Theme/Search.razor @@ -0,0 +1,61 @@ +@namespace Oqtane.Themes.Controls +@using System.Net +@using Microsoft.AspNetCore.Http +@inherits ThemeControlBase +@inject IStringLocalizer Localizer +@inject NavigationManager NavigationManager +@inject IHttpContextAccessor HttpContext + +@if (_searchResultsPage != null) +{ + +
+ + + +
+ +} + + + +@code { + private Page _searchResultsPage; + private string _keywords = ""; + + [Parameter] + public string CssClass { get; set; } + + [Parameter] + public string SearchResultPagePath { get; set; } = "search"; + + protected override void OnInitialized() + { + if(!string.IsNullOrEmpty(SearchResultPagePath)) + { + _searchResultsPage = PageState.Pages.FirstOrDefault(i => i.Path == SearchResultPagePath); + } + } + + protected override void OnParametersSet() + { + } + + private void PerformSearch() + { + var keywords = HttpContext.HttpContext.Request.Form["keywords"]; + if (!string.IsNullOrEmpty(keywords) && _searchResultsPage != null) + { + var url = NavigateUrl(_searchResultsPage.Path, $"q={keywords}"); + NavigationManager.NavigateTo(url); + } + } +} + + diff --git a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor index 261597aa..936c823e 100644 --- a/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor +++ b/Oqtane.Client/Themes/OqtaneTheme/Themes/Default.razor @@ -6,7 +6,12 @@
diff --git a/Oqtane.Server/Controllers/SearchResultsController.cs b/Oqtane.Server/Controllers/SearchResultsController.cs new file mode 100644 index 00000000..4b72a464 --- /dev/null +++ b/Oqtane.Server/Controllers/SearchResultsController.cs @@ -0,0 +1,41 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Oqtane.Documentation; +using Oqtane.Enums; +using Oqtane.Infrastructure; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Controllers +{ + [Route(ControllerRoutes.ApiRoute)] + public class SearchResultsController : ModuleControllerBase + { + private readonly ISearchService _searchService; + + public SearchResultsController(ISearchService searchService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor) + { + _searchService = searchService; + } + + [HttpPost] + [Authorize(Policy = PolicyNames.ViewModule)] + public async Task Post([FromBody] Models.SearchQuery searchQuery) + { + try + { + return await _searchService.SearchAsync(searchQuery); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery); + HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return null; + } + } + } +} diff --git a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs index 19eb4cbe..d018a4bc 100644 --- a/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs +++ b/Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs @@ -21,6 +21,7 @@ using Oqtane.Infrastructure; using Oqtane.Infrastructure.Interfaces; using Oqtane.Managers; using Oqtane.Modules; +using Oqtane.Providers; using Oqtane.Repository; using Oqtane.Security; using Oqtane.Services; @@ -97,6 +98,9 @@ namespace Microsoft.Extensions.DependencyInjection services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); return services; } @@ -131,6 +135,7 @@ namespace Microsoft.Extensions.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); // managers services.AddTransient(); diff --git a/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs new file mode 100644 index 00000000..8ad3e194 --- /dev/null +++ b/Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs @@ -0,0 +1,90 @@ +using System; +using System.Linq; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Infrastructure +{ + public class SearchIndexJob : HostedServiceBase + { + private const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime"; + + public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory) + { + Name = "Search Index Job"; + Frequency = "m"; // run every minute. + Interval = 1; + IsEnabled = true; + } + + public override string ExecuteJob(IServiceProvider provider) + { + // get services + var siteRepository = provider.GetRequiredService(); + var settingRepository = provider.GetRequiredService(); + var logRepository = provider.GetRequiredService(); + var searchService = provider.GetRequiredService(); + + var sites = siteRepository.GetSites().ToList(); + var logs = new StringBuilder(); + + foreach (var site in sites) + { + var startTime = GetSearchStartTime(site.SiteId, settingRepository); + logs.AppendLine($"Search: Begin index site: {site.Name}
"); + var currentTime = DateTime.UtcNow; + + searchService.IndexContent(site.SiteId, startTime, logNote => + { + logs.AppendLine(logNote); + }, handleError => + { + logs.AppendLine(handleError); + }); + + UpdateSearchStartTime(site.SiteId, currentTime, settingRepository); + + logs.AppendLine($"Search: End index site: {site.Name}
"); + } + + return logs.ToString(); + } + + private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); + if(setting == null) + { + return null; + } + + return Convert.ToDateTime(setting.SettingValue); + } + + private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository) + { + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName); + if (setting == null) + { + setting = new Setting + { + EntityName = EntityNames.Site, + EntityId = siteId, + SettingName = SearchIndexStartTimeSettingName, + SettingValue = Convert.ToString(startTime), + }; + + settingRepository.AddSetting(setting); + } + else + { + setting.SettingValue = Convert.ToString(startTime); + settingRepository.UpdateSetting(setting); + } + } + } +} diff --git a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs index 7cf27808..447678ed 100644 --- a/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs +++ b/Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs @@ -133,6 +133,30 @@ namespace Oqtane.SiteTemplates } } }); + _pageTemplates.Add(new PageTemplate + { + Name = "Search Results", + Parent = "", + Order = 7, + Path = "search", + Icon = "oi oi-magnifying-glass", + IsNavigation = false, + IsPersonalizable = false, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + }, + PageTemplateModules = new List { + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + } + } + } + }); if (System.IO.File.Exists(Path.Combine(_environment.WebRootPath, "images", "logo-white.png"))) { diff --git a/Oqtane.Server/Infrastructure/UpgradeManager.cs b/Oqtane.Server/Infrastructure/UpgradeManager.cs index 3f0dd0f6..8737dd65 100644 --- a/Oqtane.Server/Infrastructure/UpgradeManager.cs +++ b/Oqtane.Server/Infrastructure/UpgradeManager.cs @@ -66,6 +66,9 @@ namespace Oqtane.Infrastructure case "5.1.0": Upgrade_5_1_0(tenant, scope); break; + case "5.2.0": + Upgrade_5_2_0(tenant, scope); + break; } } } @@ -385,5 +388,48 @@ namespace Oqtane.Infrastructure } } + + private void Upgrade_5_2_0(Tenant tenant, IServiceScope scope) + { + CreateSearchResultsPages(tenant, scope); + } + + private void CreateSearchResultsPages(Tenant tenant, IServiceScope scope) + { + var pageTemplates = new List(); + pageTemplates.Add(new PageTemplate + { + Name = "Search Results", + Parent = "", + Path = "search", + Icon = "oi oi-magnifying-glass", + IsNavigation = false, + IsPersonalizable = false, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + }, + PageTemplateModules = new List { + new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default, + PermissionList = new List { + new Permission(PermissionNames.View, RoleNames.Everyone, true), + new Permission(PermissionNames.View, RoleNames.Admin, true), + new Permission(PermissionNames.Edit, RoleNames.Admin, true) + } + } + } + }); + + var pages = scope.ServiceProvider.GetRequiredService(); + var sites = scope.ServiceProvider.GetRequiredService(); + foreach (var site in sites.GetSites().ToList()) + { + if (!pages.GetPages(site.SiteId).ToList().Where(item => item.Path == "search").Any()) + { + sites.CreatePages(site, pageTemplates, null); + } + } + } } } diff --git a/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs new file mode 100644 index 00000000..b68c993f --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchIndexManager : SearchIndexManagerBase + { + public const int ModuleSearchIndexManagerPriority = 200; + + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageModuleRepository _pageModuleRepostory; + private readonly IPageRepository _pageRepository; + + public ModuleSearchIndexManager( + IServiceProvider serviceProvider, + IPageModuleRepository pageModuleRepostory, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageModuleRepostory = pageModuleRepostory; + _pageRepository = pageRepository; + } + + public override string Name => EntityNames.Module; + + public override int Priority => ModuleSearchIndexManagerPriority; + + public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) + { + var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId); + var searchContentList = new List(); + + foreach(var pageModule in pageModules) + { + var page = _pageRepository.GetPage(pageModule.PageId); + if(page == null || SearchUtils.IsSystemPage(page)) + { + continue; + } + + var module = pageModule.Module; + if (module.ModuleDefinition.ServerManagerType != "") + { + _logger.LogDebug($"Search: Begin index module {module.ModuleId}."); + var type = Type.GetType(module.ModuleDefinition.ServerManagerType); + if (type?.GetInterface(nameof(ISearchable)) != null) + { + try + { + var moduleSearch = (ISearchable)ActivatorUtilities.CreateInstance(_serviceProvider, type); + var contentList = moduleSearch.GetSearchContentList(module, startTime.GetValueOrDefault(DateTime.MinValue)); + if(contentList != null) + { + foreach(var searchContent in contentList) + { + SaveModuleMetaData(searchContent, pageModule); + + searchContentList.Add(searchContent); + } + } + + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index module {module.ModuleId} failed."); + handleError($"Search: Index module {module.ModuleId} failed: {ex.Message}"); + } + } + _logger.LogDebug($"Search: End index module {module.ModuleId}."); + } + } + + processSearchContent(searchContentList); + + return searchContentList.Count; + } + + private void SaveModuleMetaData(SearchContent searchContent, PageModule pageModule) + { + searchContent.SiteId = pageModule.Module.SiteId; + + if(string.IsNullOrEmpty(searchContent.EntityName)) + { + searchContent.EntityName = EntityNames.Module; + } + + if(searchContent.EntityId == 0) + { + searchContent.EntityId = pageModule.ModuleId; + } + + if (searchContent.IsActive) + { + searchContent.IsActive = !pageModule.Module.IsDeleted; + } + + if (searchContent.ModifiedTime == DateTime.MinValue) + { + searchContent.ModifiedTime = pageModule.ModifiedOn; + } + + if (string.IsNullOrEmpty(searchContent.AdditionalContent)) + { + searchContent.AdditionalContent = string.Empty; + } + + var page = _pageRepository.GetPage(pageModule.PageId); + + if (string.IsNullOrEmpty(searchContent.Url) && page != null) + { + searchContent.Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + + if (string.IsNullOrEmpty(searchContent.Title) && page != null) + { + searchContent.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name; + } + + if (searchContent.Properties == null) + { + searchContent.Properties = new List(); + } + + if(!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() }); + } + + if (!searchContent.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName)) + { + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() }); + } + } + } +} diff --git a/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs new file mode 100644 index 00000000..c40f5f0b --- /dev/null +++ b/Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs @@ -0,0 +1,60 @@ +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class ModuleSearchResultManager : ISearchResultManager + { + public string Name => EntityNames.Module; + + private readonly IServiceProvider _serviceProvider; + + public ModuleSearchResultManager( + IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public string GetUrl(SearchResult searchResult, SearchQuery searchQuery) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)) + { + var page = pageRepository.GetPage(pageId); + if (page != null) + { + return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}"; + } + } + + return string.Empty; + } + + public bool Visible(SearchContent searchResult, SearchQuery searchQuery) + { + var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)) + { + return CanViewPage(pageId, searchQuery.User); + } + + return false; + } + + private bool CanViewPage(int pageId, User user) + { + var pageRepository = _serviceProvider.GetRequiredService(); + var page = pageRepository.GetPage(pageId); + + return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList) + && (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList)); + } + } +} diff --git a/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs new file mode 100644 index 00000000..49599804 --- /dev/null +++ b/Oqtane.Server/Managers/Search/PageSearchIndexManager.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Mvc.RazorPages; +using System.Reflection.Metadata; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Interfaces; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public class PageSearchIndexManager : SearchIndexManagerBase + { + private const int PageSearchIndexManagerPriority = 100; + + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private readonly IPageRepository _pageRepository; + + public PageSearchIndexManager( + IServiceProvider serviceProvider, + ILogger logger, + IPageRepository pageRepository) + : base(serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = logger; + _pageRepository = pageRepository; + } + + public override string Name => EntityNames.Page; + + public override int Priority => PageSearchIndexManagerPriority; + + public override int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError) + { + var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue); + var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue); + var searchContentList = new List(); + + foreach(var page in pages) + { + try + { + if(SearchUtils.IsSystemPage(page)) + { + continue; + } + + var searchContent = new SearchContent + { + EntityName = EntityNames.Page, + EntityId = page.PageId, + SiteId = page.SiteId, + ModifiedTime = page.ModifiedOn, + AdditionalContent = string.Empty, + Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}", + Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name, + Description = string.Empty, + Body = $"{page.Name} {page.Title}", + IsActive = !page.IsDeleted && Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) + }; + + if (searchContent.Properties == null) + { + searchContent.Properties = new List(); + } + + if (!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName)) + { + searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() }); + } + + searchContentList.Add(searchContent); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Index page {page.PageId} failed."); + handleError($"Search: Index page {page.PageId} failed: {ex.Message}"); + } + } + + processSearchContent(searchContentList); + + return searchContentList.Count; + } + } +} diff --git a/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs new file mode 100644 index 00000000..779a950d --- /dev/null +++ b/Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; + +namespace Oqtane.Managers.Search +{ + public abstract class SearchIndexManagerBase : ISearchIndexManager + { + private const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled"; + + private readonly IServiceProvider _serviceProvider; + + public SearchIndexManagerBase(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public abstract int Priority { get; } + + public abstract string Name { get; } + + public abstract int IndexContent(int siteId, DateTime? startDate, Action> processSearchContent, Action handleError); + + public virtual bool IsIndexEnabled(int siteId) + { + var settingName = string.Format(SearchIndexManagerEnabledSettingFormat, Name); + var settingRepository = _serviceProvider.GetRequiredService(); + var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName); + return setting == null || setting.SettingValue == "true"; + } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs new file mode 100644 index 00000000..4cf2aa4a --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs @@ -0,0 +1,61 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; +using Oqtane.Models; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentEntityBuilder : AuditableBaseEntityBuilder + { + private const string _entityTableName = "SearchContent"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContent", x => x.SearchContentId); + + public SearchContentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + } + + protected override SearchContentEntityBuilder BuildTable(ColumnsBuilder table) + { + SearchContentId = AddAutoIncrementColumn(table, "SearchContentId"); + EntityName = AddStringColumn(table, "EntityName", 50); + EntityId = AddIntegerColumn(table, "EntityId"); + SiteId = AddIntegerColumn(table, "SiteId"); + Title = AddStringColumn(table, "Title", 255); + Description = AddMaxStringColumn(table, "Description"); + Body = AddMaxStringColumn(table, "Body"); + Url = AddStringColumn(table, "Url", 255); + ModifiedTime = AddDateTimeColumn(table, "ModifiedTime"); + IsActive = AddBooleanColumn(table, "IsActive"); + AdditionalContent = AddMaxStringColumn(table, "AdditionalContent"); + + AddAuditableColumns(table); + + return this; + } + + public OperationBuilder SearchContentId { get; private set; } + + public OperationBuilder EntityName { get; private set; } + + public OperationBuilder EntityId { get; private set; } + + public OperationBuilder SiteId { get; private set; } + + public OperationBuilder Title { get; private set; } + + public OperationBuilder Description { get; private set; } + + public OperationBuilder Body { get; private set; } + + public OperationBuilder Url { get; private set; } + + public OperationBuilder ModifiedTime { get; private set; } + + public OperationBuilder IsActive { get; private set; } + + public OperationBuilder AdditionalContent { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs new file mode 100644 index 00000000..01870cd1 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentPropertyEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentProperty"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentProperty", x => x.PropertyId); + private readonly ForeignKey _searchContentForeignKey = new("FK_SearchContentProperty_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade); + + public SearchContentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchContentForeignKey); + } + + protected override SearchContentPropertyEntityBuilder BuildTable(ColumnsBuilder table) + { + PropertyId = AddAutoIncrementColumn(table, "PropertyId"); + SearchContentId = AddIntegerColumn(table, "SearchContentId"); + Name = AddStringColumn(table, "Name", 50); + Value = AddStringColumn(table, "Value", 50); + + return this; + } + + public OperationBuilder PropertyId { get; private set; } + + public OperationBuilder SearchContentId { get; private set; } + + public OperationBuilder Name { get; private set; } + + public OperationBuilder Value { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs new file mode 100644 index 00000000..2eb845fc --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs @@ -0,0 +1,42 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentWordsEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentWords"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentWords", x => x.WordId); + private readonly ForeignKey _searchContentForeignKey = new("FK_SearchContentWords_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade); + private readonly ForeignKey _wordSourceForeignKey = new("FK_SearchContentWords_WordSource", x => x.WordSourceId, "SearchContentWordSource", "WordSourceId", ReferentialAction.Cascade); + + public SearchContentWordsEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + + ForeignKeys.Add(_searchContentForeignKey); + ForeignKeys.Add(_wordSourceForeignKey); + } + + protected override SearchContentWordsEntityBuilder BuildTable(ColumnsBuilder table) + { + WordId = AddAutoIncrementColumn(table, "WordId"); + SearchContentId = AddIntegerColumn(table, "SearchContentId"); + WordSourceId = AddIntegerColumn(table, "WordSourceId"); + Count = AddIntegerColumn(table, "Count"); + + return this; + } + + public OperationBuilder WordId { get; private set; } + + public OperationBuilder SearchContentId { get; private set; } + + public OperationBuilder WordSourceId { get; private set; } + + public OperationBuilder Count { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs new file mode 100644 index 00000000..0787a3a4 --- /dev/null +++ b/Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders; +using Oqtane.Databases.Interfaces; + +namespace Oqtane.Migrations.EntityBuilders +{ + public class SearchContentWordSourceEntityBuilder : BaseEntityBuilder + { + private const string _entityTableName = "SearchContentWordSource"; + private readonly PrimaryKey _primaryKey = new("PK_SearchContentWordSource", x => x.WordSourceId); + + public SearchContentWordSourceEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database) + { + EntityTableName = _entityTableName; + PrimaryKey = _primaryKey; + } + + protected override SearchContentWordSourceEntityBuilder BuildTable(ColumnsBuilder table) + { + WordSourceId = AddAutoIncrementColumn(table, "WordSourceId"); + Word = AddStringColumn(table, "Word", 255); + + return this; + } + + public OperationBuilder WordSourceId { get; private set; } + + public OperationBuilder Word { get; private set; } + } +} diff --git a/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs new file mode 100644 index 00000000..c66a4fe5 --- /dev/null +++ b/Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs @@ -0,0 +1,50 @@ +using System; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Oqtane.Databases.Interfaces; +using Oqtane.Migrations.EntityBuilders; +using Oqtane.Repository; + +namespace Oqtane.Migrations.Tenant +{ + [DbContext(typeof(TenantDBContext))] + [Migration("Tenant.05.02.00.01")] + public class AddSearchTables : MultiDatabaseMigration + { + public AddSearchTables(IDatabase database) : base(database) + { + } + + protected override void Up(MigrationBuilder migrationBuilder) + { + var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentEntityBuilder.Create(); + + var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentPropertyEntityBuilder.Create(); + + var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordSourceEntityBuilder.Create(); + searchContentWordSourceEntityBuilder.AddIndex("IX_SearchContentWordSource", "Word", true); + + var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordsEntityBuilder.Create(); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordsEntityBuilder.Drop(); + + var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentWordSourceEntityBuilder.DropIndex("IX_SearchContentWordSource"); + searchContentWordSourceEntityBuilder.Drop(); + + var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentPropertyEntityBuilder.Drop(); + + var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase); + searchContentEntityBuilder.Drop(); + } + } +} diff --git a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs index 6c9293fb..e5cfb8de 100644 --- a/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs +++ b/Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs @@ -8,20 +8,29 @@ using Oqtane.Shared; using Oqtane.Migrations.Framework; using Oqtane.Documentation; using System.Linq; +using Oqtane.Interfaces; +using System.Collections.Generic; +using System; // ReSharper disable ConvertToUsingDeclaration namespace Oqtane.Modules.HtmlText.Manager { [PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")] - public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable + public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, ISearchable { + private readonly IServiceProvider _serviceProvider; private readonly IHtmlTextRepository _htmlText; private readonly IDBContextDependencies _DBContextDependencies; private readonly ISqlRepository _sqlRepository; - public HtmlTextManager(IHtmlTextRepository htmlText, IDBContextDependencies DBContextDependencies, ISqlRepository sqlRepository) + public HtmlTextManager( + IServiceProvider serviceProvider, + IHtmlTextRepository htmlText, + IDBContextDependencies DBContextDependencies, + ISqlRepository sqlRepository) { + _serviceProvider = serviceProvider; _htmlText = htmlText; _DBContextDependencies = DBContextDependencies; _sqlRepository = sqlRepository; @@ -39,6 +48,27 @@ namespace Oqtane.Modules.HtmlText.Manager return content; } + public IList GetSearchContentList(Module module, DateTime startDate) + { + var searchContentList = new List(); + + var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId); + if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate)) + { + var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First(); + + searchContentList.Add(new SearchContent + { + Title = module.Title, + Description = string.Empty, + Body = htmltext.Content, + ModifiedTime = htmltext.ModifiedOn + }); + } + + return searchContentList; + } + public void ImportModule(Module module, string content, string version) { content = WebUtility.HtmlDecode(content); diff --git a/Oqtane.Server/Oqtane.Server.csproj b/Oqtane.Server/Oqtane.Server.csproj index f661e715..52776903 100644 --- a/Oqtane.Server/Oqtane.Server.csproj +++ b/Oqtane.Server/Oqtane.Server.csproj @@ -33,6 +33,7 @@ + diff --git a/Oqtane.Server/Providers/DatabaseSearchProvider.cs b/Oqtane.Server/Providers/DatabaseSearchProvider.cs new file mode 100644 index 00000000..adf2cee8 --- /dev/null +++ b/Oqtane.Server/Providers/DatabaseSearchProvider.cs @@ -0,0 +1,308 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml; +using HtmlAgilityPack; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Services; +using Oqtane.Shared; +using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; + +namespace Oqtane.Providers +{ + public class DatabaseSearchProvider : ISearchProvider + { + private readonly ISearchContentRepository _searchContentRepository; + + private const string IgnoreWords = "the,be,to,of,and,a,i,in,that,have,it,for,not,on,with,he,as,you,do,at,this,but,his,by,from,they,we,say,her,she,or,an,will,my,one,all,would,there,their,what,so,up,out,if,about,who,get,which,go,me,when,make,can,like,time,no,just,him,know,take,people,into,year,your,good,some,could,them,see,other,than,then,now,look,only,come,its,over,think,also,back,after,use,two,how,our,work,first,well,way,even,new,want,because,any,these,give,day,most,us"; + private const int WordMinLength = 3; + public string Name => Constants.DefaultSearchProviderName; + + public DatabaseSearchProvider(ISearchContentRepository searchContentRepository) + { + _searchContentRepository = searchContentRepository; + } + + public void Commit() + { + } + + public void DeleteSearchContent(string id) + { + _searchContentRepository.DeleteSearchContent(id); + } + + public bool Optimize() + { + return true; + } + + public void ResetIndex() + { + _searchContentRepository.DeleteAllSearchContent(); + } + + public void SaveSearchContent(SearchContent searchContent, bool autoCommit = false) + { + //remove exist document + _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId); + + //clean the search content to remove html tags + CleanSearchContent(searchContent); + + _searchContentRepository.AddSearchContent(searchContent); + + //save the index words + AnalyzeSearchContent(searchContent); + } + + public async Task SearchAsync(SearchQuery searchQuery, Func validateFunc) + { + var totalResults = 0; + + var searchContentList = await _searchContentRepository.GetSearchContentListAsync(searchQuery); + + //convert the search content to search results. + var results = searchContentList + .Where(i => validateFunc(i, searchQuery)) + .Select(i => ConvertToSearchResult(i, searchQuery)); + + if (searchQuery.SortDirection == SearchSortDirections.Descending) + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderByDescending(i => i.ModifiedTime); + break; + } + } + else + { + switch (searchQuery.SortField) + { + case SearchSortFields.Relevance: + results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ModifiedTime); + break; + case SearchSortFields.Title: + results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ModifiedTime); + break; + default: + results = results.OrderBy(i => i.ModifiedTime); + break; + } + } + + //remove duplicated results based on page id for Page and Module types + results = results.DistinctBy(i => + { + if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module) + { + var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty; + return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey; + } + else + { + return i.UniqueKey; + } + }); + + totalResults = results.Count(); + + return new SearchResults + { + Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(), + TotalResults = totalResults + }; + } + + private SearchResult ConvertToSearchResult(SearchContent searchContent, SearchQuery searchQuery) + { + var searchResult = new SearchResult() + { + SearchContentId = searchContent.SearchContentId, + SiteId = searchContent.SiteId, + EntityName = searchContent.EntityName, + EntityId = searchContent.EntityId, + Title = searchContent.Title, + Description = searchContent.Description, + Body = searchContent.Body, + Url = searchContent.Url, + ModifiedTime = searchContent.ModifiedTime, + Properties = searchContent.Properties, + Snippet = BuildSnippet(searchContent, searchQuery), + Score = CalculateScore(searchContent, searchQuery) + }; + + return searchResult; + } + + private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery) + { + var score = 0f; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + score += searchContent.Words.Where(i => i.WordSource.Word.StartsWith(keyword)).Sum(i => i.Count); + } + + return score / 100; + } + + private string BuildSnippet(SearchContent searchContent, SearchQuery searchQuery) + { + var content = $"{searchContent.Title} {searchContent.Description} {searchContent.Body}"; + var snippet = string.Empty; + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + if (!string.IsNullOrWhiteSpace(keyword) && content.Contains(keyword, StringComparison.OrdinalIgnoreCase)) + { + var start = content.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) - 20; + var prefix = "..."; + var suffix = "..."; + if (start <= 0) + { + start = 0; + prefix = string.Empty; + } + + var length = searchQuery.BodySnippetLength; + if (start + length >= content.Length) + { + length = content.Length - start; + suffix = string.Empty; + } + + snippet = $"{prefix}{content.Substring(start, length)}{suffix}"; + break; + + + } + } + + if (string.IsNullOrEmpty(snippet)) + { + snippet = content.Substring(0, searchQuery.BodySnippetLength); + } + + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + snippet = Regex.Replace(snippet, $"({keyword})", $"$1", RegexOptions.IgnoreCase); + } + + return snippet; + } + + private void AnalyzeSearchContent(SearchContent searchContent) + { + //analyze the search content and save the index words + var indexContent = $"{searchContent.Title} {searchContent.Description} {searchContent.Body} {searchContent.AdditionalContent}"; + var words = GetWords(indexContent, WordMinLength); + var existWords = _searchContentRepository.GetWords(searchContent.SearchContentId); + foreach (var kvp in words) + { + var word = existWords.FirstOrDefault(i => i.WordSource.Word == kvp.Key); + if (word != null) + { + word.Count = kvp.Value; + _searchContentRepository.UpdateSearchContentWords(word); + } + else + { + var wordSource = _searchContentRepository.GetSearchContentWordSource(kvp.Key); + if (wordSource == null) + { + wordSource = _searchContentRepository.AddSearchContentWordSource(new SearchContentWordSource { Word = kvp.Key }); + } + + word = new SearchContentWords + { + SearchContentId = searchContent.SearchContentId, + WordSourceId = wordSource.WordSourceId, + Count = kvp.Value + }; + + _searchContentRepository.AddSearchContentWords(word); + } + } + } + + private static Dictionary GetWords(string content, int minLength) + { + content = FormatText(content); + + var words = new Dictionary(); + var ignoreWords = IgnoreWords.Split(','); + + if (!string.IsNullOrEmpty(content)) + { + foreach (var word in content.Split(' ')) + { + if (word.Length >= minLength && !ignoreWords.Contains(word)) + { + if (!words.ContainsKey(word)) + { + words.Add(word, 1); + } + else + { + words[word] += 1; + } + } + } + } + + return words; + } + + private static string FormatText(string text) + { + text = HtmlEntity.DeEntitize(text); + foreach (var punctuation in ".?!,;:-_()[]{}'\"/\\".ToCharArray()) + { + text = text.Replace(punctuation, ' '); + } + text = text.Replace(" ", " ").ToLower().Trim(); + + return text; + } + + private void CleanSearchContent(SearchContent searchContent) + { + searchContent.Title = GetCleanContent(searchContent.Title); + searchContent.Description = GetCleanContent(searchContent.Description); + searchContent.Body = GetCleanContent(searchContent.Body); + searchContent.AdditionalContent = GetCleanContent(searchContent.AdditionalContent); + } + + private string GetCleanContent(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return string.Empty; + } + + content = WebUtility.HtmlDecode(content); + + var page = new HtmlDocument(); + page.LoadHtml(content); + + var phrases = page.DocumentNode.Descendants().Where(i => + i.NodeType == HtmlNodeType.Text && + i.ParentNode.Name != "script" && + i.ParentNode.Name != "style" && + !string.IsNullOrEmpty(i.InnerText.Trim()) + ).Select(i => i.InnerText); + + return string.Join(" ", phrases); + } + } +} diff --git a/Oqtane.Server/Repository/Context/TenantDBContext.cs b/Oqtane.Server/Repository/Context/TenantDBContext.cs index 2efad3f7..cb0dcb28 100644 --- a/Oqtane.Server/Repository/Context/TenantDBContext.cs +++ b/Oqtane.Server/Repository/Context/TenantDBContext.cs @@ -29,5 +29,9 @@ namespace Oqtane.Repository public virtual DbSet Language { get; set; } public virtual DbSet Visitor { get; set; } public virtual DbSet UrlMapping { get; set; } + public virtual DbSet SearchContent { get; set; } + public virtual DbSet SearchContentProperty { get; set; } + public virtual DbSet SearchContentWords { get; set; } + public virtual DbSet SearchContentWordSource { get; set; } } } diff --git a/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs b/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs new file mode 100644 index 00000000..868265b0 --- /dev/null +++ b/Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Repository +{ + public interface ISearchContentRepository + { + Task> GetSearchContentListAsync(SearchQuery searchQuery); + SearchContent AddSearchContent(SearchContent searchContent); + void DeleteSearchContent(int searchContentId); + void DeleteSearchContent(string entityName, int entryId); + void DeleteSearchContent(string uniqueKey); + void DeleteAllSearchContent(); + + SearchContentWordSource GetSearchContentWordSource(string word); + SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource); + + IEnumerable GetWords(int searchContentId); + SearchContentWords AddSearchContentWords(SearchContentWords word); + SearchContentWords UpdateSearchContentWords(SearchContentWords word); + } +} diff --git a/Oqtane.Server/Repository/SearchContentRepository.cs b/Oqtane.Server/Repository/SearchContentRepository.cs new file mode 100644 index 00000000..67ecfcf7 --- /dev/null +++ b/Oqtane.Server/Repository/SearchContentRepository.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Oqtane.Models; +using Oqtane.Shared; + +namespace Oqtane.Repository +{ + public class SearchContentRepository : ISearchContentRepository + { + private readonly IDbContextFactory _dbContextFactory; + + public SearchContentRepository(IDbContextFactory dbContextFactory) + { + _dbContextFactory = dbContextFactory; + } + + public async Task> GetSearchContentListAsync(SearchQuery searchQuery) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContentList = db.SearchContent.AsNoTracking() + .Include(i => i.Properties) + .Include(i => i.Words) + .ThenInclude(w => w.WordSource) + .Where(i => i.SiteId == searchQuery.SiteId && i.IsActive); + + if (searchQuery.EntityNames != null && searchQuery.EntityNames.Any()) + { + searchContentList = searchContentList.Where(i => searchQuery.EntityNames.Contains(i.EntityName)); + } + + if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue) + { + searchContentList = searchContentList.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc); + } + + if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue) + { + searchContentList = searchContentList.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc); + } + + if (searchQuery.Properties != null && searchQuery.Properties.Any()) + { + foreach (var property in searchQuery.Properties) + { + searchContentList = searchContentList.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value)); + } + } + + var filteredContentList = new List(); + if (!string.IsNullOrEmpty(searchQuery.Keywords)) + { + foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords)) + { + filteredContentList.AddRange(await searchContentList.Where(i => i.Words.Any(w => w.WordSource.Word.StartsWith(keyword))).ToListAsync()); + } + } + + return filteredContentList.DistinctBy(i => i.UniqueKey); + } + + public SearchContent AddSearchContent(SearchContent searchContent) + { + using var context = _dbContextFactory.CreateDbContext(); + context.SearchContent.Add(searchContent); + + if(searchContent.Properties != null && searchContent.Properties.Any()) + { + foreach(var property in searchContent.Properties) + { + property.SearchContentId = searchContent.SearchContentId; + context.SearchContentProperty.Add(property); + } + } + + context.SaveChanges(); + + return searchContent; + } + + public void DeleteSearchContent(int searchContentId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.Find(searchContentId); + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + + public void DeleteSearchContent(string entityName, int entryId) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.FirstOrDefault(i => i.EntityName == entityName && i.EntityId == entryId); + if(searchContent != null) + { + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + } + + public void DeleteSearchContent(string uniqueKey) + { + using var db = _dbContextFactory.CreateDbContext(); + var searchContent = db.SearchContent.FirstOrDefault(i => (i.EntityName + ":" + i.EntityId) == uniqueKey); + if (searchContent != null) + { + db.SearchContent.Remove(searchContent); + db.SaveChanges(); + } + } + + public void DeleteAllSearchContent() + { + using var db = _dbContextFactory.CreateDbContext(); + db.SearchContent.RemoveRange(db.SearchContent); + db.SaveChanges(); + } + + public SearchContentWordSource GetSearchContentWordSource(string word) + { + if(string.IsNullOrEmpty(word)) + { + return null; + } + + using var db = _dbContextFactory.CreateDbContext(); + return db.SearchContentWordSource.FirstOrDefault(i => i.Word == word); + } + + public SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.SearchContentWordSource.Add(wordSource); + db.SaveChanges(); + + return wordSource; + } + + public IEnumerable GetWords(int searchContentId) + { + using var db = _dbContextFactory.CreateDbContext(); + return db.SearchContentWords + .Include(i => i.WordSource) + .Where(i => i.SearchContentId == searchContentId).ToList(); + } + + public SearchContentWords AddSearchContentWords(SearchContentWords word) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.SearchContentWords.Add(word); + db.SaveChanges(); + + return word; + } + + public SearchContentWords UpdateSearchContentWords(SearchContentWords word) + { + using var db = _dbContextFactory.CreateDbContext(); + + db.Entry(word).State = EntityState.Modified; + db.SaveChanges(); + + return word; + } + } +} diff --git a/Oqtane.Server/Services/SearchService.cs b/Oqtane.Server/Services/SearchService.cs new file mode 100644 index 00000000..8d3fe4ef --- /dev/null +++ b/Oqtane.Server/Services/SearchService.cs @@ -0,0 +1,234 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oqtane.Infrastructure; +using Oqtane.Models; +using Oqtane.Repository; +using Oqtane.Security; +using Oqtane.Shared; + +namespace Oqtane.Services +{ + public class SearchService : ISearchService + { + private const string SearchProviderSettingName = "SearchProvider"; + private const string SearchEnabledSettingName = "SearchEnabled"; + + private readonly IServiceProvider _serviceProvider; + private readonly ITenantManager _tenantManager; + private readonly IAliasRepository _aliasRepository; + private readonly ISettingRepository _settingRepository; + private readonly IPermissionRepository _permissionRepository; + private readonly ILogger _logger; + private readonly IMemoryCache _cache; + + public SearchService( + IServiceProvider serviceProvider, + ITenantManager tenantManager, + IAliasRepository aliasRepository, + ISettingRepository settingRepository, + IPermissionRepository permissionRepository, + ILogger logger, + IMemoryCache cache) + { + _tenantManager = tenantManager; + _aliasRepository = aliasRepository; + _settingRepository = settingRepository; + _permissionRepository = permissionRepository; + _serviceProvider = serviceProvider; + _logger = logger; + _cache = cache; + } + + public void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError) + { + var searchEnabled = SearchEnabled(siteId); + if(!searchEnabled) + { + logNote($"Search: Search is disabled on site {siteId}.
"); + return; + } + + _logger.LogDebug($"Search: Start Index Content of {siteId}, Start Time: {startTime.GetValueOrDefault(DateTime.MinValue)}"); + + var searchProvider = GetSearchProvider(siteId); + + SetTenant(siteId); + + if (startTime == null) + { + searchProvider.ResetIndex(); + } + + var searchIndexManagers = GetSearchIndexManagers(m => { }); + foreach (var searchIndexManager in searchIndexManagers) + { + if (!searchIndexManager.IsIndexEnabled(siteId)) + { + logNote($"Search: Ignore indexer {searchIndexManager.Name} because it's disabled.
"); + } + else + { + _logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}"); + + var count = searchIndexManager.IndexContent(siteId, startTime, SaveSearchContent, handleError); + logNote($"Search: Indexer {searchIndexManager.Name} processed {count} search content.
"); + + _logger.LogDebug($"Search: End Index {searchIndexManager.Name}"); + } + } + } + + public async Task SearchAsync(SearchQuery searchQuery) + { + var searchProvider = GetSearchProvider(searchQuery.SiteId); + var searchResults = await searchProvider.SearchAsync(searchQuery, Visible); + + //generate the document url if it's not set. + foreach (var result in searchResults.Results) + { + if(string.IsNullOrEmpty(result.Url)) + { + result.Url = GetDocumentUrl(result, searchQuery); + } + } + + return searchResults; + } + + private ISearchProvider GetSearchProvider(int siteId) + { + var providerName = GetSearchProviderSetting(siteId); + var searchProviders = _serviceProvider.GetServices(); + var provider = searchProviders.FirstOrDefault(i => i.Name == providerName); + if(provider == null) + { + provider = searchProviders.FirstOrDefault(i => i.Name == Constants.DefaultSearchProviderName); + } + + return provider; + } + + private string GetSearchProviderSetting(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchProviderSettingName); + if(!string.IsNullOrEmpty(setting?.SettingValue)) + { + return setting.SettingValue; + } + + return Constants.DefaultSearchProviderName; + } + + private bool SearchEnabled(int siteId) + { + var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchEnabledSettingName); + if (!string.IsNullOrEmpty(setting?.SettingValue)) + { + return bool.TryParse(setting.SettingValue, out bool enabled) && enabled; + } + + return true; + } + + private void SetTenant(int siteId) + { + var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId); + _tenantManager.SetAlias(alias); + } + + private IList GetSearchIndexManagers(Action initManager) + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchIndexManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchIndexManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + initManager(manager); + managers.Add(manager); + } + + return managers.OrderBy(i => i.Priority).ToList(); + } + + private IList GetSearchResultManagers() + { + var managers = new List(); + var managerTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract); + + foreach (var type in managerTypes) + { + var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type); + managers.Add(manager); + } + + return managers.ToList(); + } + + private void SaveSearchContent(IList searchContentList) + { + if(searchContentList.Any()) + { + var searchProvider = GetSearchProvider(searchContentList.First().SiteId); + + foreach (var searchContent in searchContentList) + { + try + { + searchProvider.SaveSearchContent(searchContent); + } + catch(Exception ex) + { + _logger.LogError(ex, $"Search: Save search content {searchContent.UniqueKey} failed."); + } + } + + //commit the index changes + searchProvider.Commit(); + } + } + + private bool Visible(SearchContent searchContent, SearchQuery searchQuery) + { + if(!HasViewPermission(searchQuery.SiteId, searchQuery.User, searchContent.EntityName, searchContent.EntityId)) + { + return false; + } + + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchContent.EntityName); + if (searchResultManager != null) + { + return searchResultManager.Visible(searchContent, searchQuery); + } + return true; + } + + private bool HasViewPermission(int siteId, User user, string entityName, int entityId) + { + var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList(); + return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions); + } + + private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery) + { + var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.EntityName); + if(searchResultManager != null) + { + return searchResultManager.GetUrl(result, searchQuery); + } + + return string.Empty; + } + } +} diff --git a/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css new file mode 100644 index 00000000..71194e38 --- /dev/null +++ b/Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css @@ -0,0 +1,3 @@ +.search-result-container ul.pagination li label, .search-result-container ul.dropdown-menu li label { + cursor: pointer; +} \ No newline at end of file diff --git a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css index 1b9ea409..bbb65908 100644 --- a/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css +++ b/Oqtane.Server/wwwroot/Themes/Oqtane.Themes.OqtaneTheme/Theme.css @@ -79,6 +79,10 @@ body { top: -2px; } +.app-search input{ + width: auto; +} + .navbar-toggler { background-color: rgba(255, 255, 255, 0.1); margin: .5rem; diff --git a/Oqtane.Server/wwwroot/css/app.css b/Oqtane.Server/wwwroot/css/app.css index f5a65a80..941ff5f1 100644 --- a/Oqtane.Server/wwwroot/css/app.css +++ b/Oqtane.Server/wwwroot/css/app.css @@ -235,3 +235,17 @@ app { .app-form-inline { display: inline-block; } +.app-search{ + display: inline-block; + position: relative; +} +.app-search input + button{ + background: none; + border: none; + position: absolute; + right: 0; + top: 0; +} +.app-search input + button .oi{ + top: 0; +} \ No newline at end of file diff --git a/Oqtane.Shared/Enums/SearchSortDirections.cs b/Oqtane.Shared/Enums/SearchSortDirections.cs new file mode 100644 index 00000000..62c7f08d --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortDirections.cs @@ -0,0 +1,10 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortDirections + { + Ascending, + Descending + } +} diff --git a/Oqtane.Shared/Enums/SearchSortFields.cs b/Oqtane.Shared/Enums/SearchSortFields.cs new file mode 100644 index 00000000..c8e16175 --- /dev/null +++ b/Oqtane.Shared/Enums/SearchSortFields.cs @@ -0,0 +1,11 @@ +using System; + +namespace Oqtane.Shared +{ + public enum SearchSortFields + { + Relevance, + Title, + LastModified + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchIndexManager.cs b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs new file mode 100644 index 00000000..0a4123f8 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchIndexManager.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Web; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchIndexManager + { + int Priority { get; } + + string Name { get; } + + bool IsIndexEnabled(int siteId); + + int IndexContent(int siteId, DateTime? startTime, Action> processSearchContent, Action handleError); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchProvider.cs b/Oqtane.Shared/Interfaces/ISearchProvider.cs new file mode 100644 index 00000000..cb075d3c --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchProvider.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchProvider + { + string Name { get; } + + void SaveSearchContent(SearchContent searchContent, bool autoCommit = false); + + void DeleteSearchContent(string id); + + Task SearchAsync(SearchQuery searchQuery, Func validateFunc); + + bool Optimize(); + + void Commit(); + + void ResetIndex(); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchResultManager.cs b/Oqtane.Shared/Interfaces/ISearchResultManager.cs new file mode 100644 index 00000000..b6f6cd32 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchResultManager.cs @@ -0,0 +1,14 @@ +using System; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchResultManager + { + string Name { get; } + + bool Visible(SearchContent searchResult, SearchQuery searchQuery); + + string GetUrl(SearchResult searchResult, SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchService.cs b/Oqtane.Shared/Interfaces/ISearchService.cs new file mode 100644 index 00000000..0f125c65 --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchService.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Services +{ + public interface ISearchService + { + void IndexContent(int siteId, DateTime? startTime, Action logNote, Action handleError); + + Task SearchAsync(SearchQuery searchQuery); + } +} diff --git a/Oqtane.Shared/Interfaces/ISearchable.cs b/Oqtane.Shared/Interfaces/ISearchable.cs new file mode 100644 index 00000000..c0a4c00e --- /dev/null +++ b/Oqtane.Shared/Interfaces/ISearchable.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Oqtane.Models; + +namespace Oqtane.Interfaces +{ + public interface ISearchable + { + public IList GetSearchContentList(Module module, DateTime startTime); + } +} diff --git a/Oqtane.Shared/Models/SearchContent.cs b/Oqtane.Shared/Models/SearchContent.cs new file mode 100644 index 00000000..36f4c664 --- /dev/null +++ b/Oqtane.Shared/Models/SearchContent.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Text.Json; +namespace Oqtane.Models +{ + public class SearchContent : ModelBase + { + public int SearchContentId { get; set; } + + [NotMapped] + public string UniqueKey => $"{EntityName}:{EntityId}"; + + public string EntityName { get; set; } + + public int EntityId { get; set; } + + public int SiteId { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Body { get; set; } + + public string Url { get; set; } + + public DateTime ModifiedTime { get; set; } + + public bool IsActive { get; set; } = true; + + public string AdditionalContent { get; set; } + + public IList Properties { get; set; } + + public IList Words { get; set; } + + public override string ToString() + { + return JsonSerializer.Serialize(this); + } + } +} diff --git a/Oqtane.Shared/Models/SearchContentProperty.cs b/Oqtane.Shared/Models/SearchContentProperty.cs new file mode 100644 index 00000000..0c2b7f9b --- /dev/null +++ b/Oqtane.Shared/Models/SearchContentProperty.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.EntityFrameworkCore; + +namespace Oqtane.Models +{ + public class SearchContentProperty + { + [Key] + public int PropertyId { get; set; } + + public int SearchContentId { get; set; } + + public string Name { get; set; } + + public string Value { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchContentWordSource.cs b/Oqtane.Shared/Models/SearchContentWordSource.cs new file mode 100644 index 00000000..ec814757 --- /dev/null +++ b/Oqtane.Shared/Models/SearchContentWordSource.cs @@ -0,0 +1,12 @@ +using System.ComponentModel.DataAnnotations; + +namespace Oqtane.Models +{ + public class SearchContentWordSource + { + [Key] + public int WordSourceId { get; set; } + + public string Word { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchContentWords.cs b/Oqtane.Shared/Models/SearchContentWords.cs new file mode 100644 index 00000000..c7630815 --- /dev/null +++ b/Oqtane.Shared/Models/SearchContentWords.cs @@ -0,0 +1,19 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace Oqtane.Models +{ + public class SearchContentWords + { + [Key] + public int WordId { get; set; } + + public int SearchContentId { get; set; } + + public int WordSourceId { get; set; } + + public int Count { get; set; } + + public SearchContentWordSource WordSource { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchQuery.cs b/Oqtane.Shared/Models/SearchQuery.cs new file mode 100644 index 00000000..cf03e8c1 --- /dev/null +++ b/Oqtane.Shared/Models/SearchQuery.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using Oqtane.Shared; + +namespace Oqtane.Models +{ + public class SearchQuery + { + public int SiteId { get; set; } + + public Alias Alias { get; set; } + + public User User { get; set; } + + public string Keywords { get; set; } + + public IList EntityNames { get; set; } = new List(); + + public DateTime BeginModifiedTimeUtc { get; set; } + + public DateTime EndModifiedTimeUtc { get; set; } + + public IDictionary Properties { get; set; } = new Dictionary(); + + public int PageIndex { get; set; } + + public int PageSize { get; set; } + + public SearchSortFields SortField { get; set; } + + public SearchSortDirections SortDirection { get; set; } + + public int BodySnippetLength { get; set;} = 255; + } +} diff --git a/Oqtane.Shared/Models/SearchResult.cs b/Oqtane.Shared/Models/SearchResult.cs new file mode 100644 index 00000000..dd878d42 --- /dev/null +++ b/Oqtane.Shared/Models/SearchResult.cs @@ -0,0 +1,11 @@ +namespace Oqtane.Models +{ + public class SearchResult : SearchContent + { + public float Score { get; set; } + + public string DisplayScore { get; set; } + + public string Snippet { get; set; } + } +} diff --git a/Oqtane.Shared/Models/SearchResults.cs b/Oqtane.Shared/Models/SearchResults.cs new file mode 100644 index 00000000..5642468c --- /dev/null +++ b/Oqtane.Shared/Models/SearchResults.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Oqtane.Models +{ + public class SearchResults + { + public IList Results { get; set; } + + public int TotalResults { get; set; } + } +} diff --git a/Oqtane.Shared/Shared/Constants.cs b/Oqtane.Shared/Shared/Constants.cs index b2b8c837..979f0872 100644 --- a/Oqtane.Shared/Shared/Constants.cs +++ b/Oqtane.Shared/Shared/Constants.cs @@ -77,6 +77,10 @@ namespace Oqtane.Shared public static readonly string VisitorCookiePrefix = "APP_VISITOR_"; + public const string DefaultSearchProviderName = "Database"; + public const string SearchPageIdPropertyName = "PageId"; + public const string SearchModuleIdPropertyName = "ModuleId"; + // Obsolete constants const string RoleObsoleteMessage = "Use the corresponding member from Oqtane.Shared.RoleNames"; diff --git a/Oqtane.Shared/Shared/SearchUtils.cs b/Oqtane.Shared/Shared/SearchUtils.cs new file mode 100644 index 00000000..d49c932d --- /dev/null +++ b/Oqtane.Shared/Shared/SearchUtils.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; + +namespace Oqtane.Shared +{ + public sealed class SearchUtils + { + private static readonly IList _systemPages; + + static SearchUtils() + { + _systemPages = new List { "login", "register", "profile", "404", "search" }; + } + + public static IList GetKeywordsList(string keywords) + { + var keywordsList = new List(); + if(!string.IsNullOrEmpty(keywords)) + { + foreach (var keyword in keywords.Split(' ')) + { + if (!string.IsNullOrWhiteSpace(keyword.Trim())) + { + keywordsList.Add(keyword.Trim().ToLower()); + } + } + } + + return keywordsList; + } + + public static bool IsSystemPage(Models.Page page) + { + return page.Path.Contains("admin") || _systemPages.Contains(page.Path); + } + } +}