Diplomarbeit-Absolventenverein/oqtane.framework
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

refactoring the code.

Browse Source
This commit is contained in:
Ben
2024-06-04 17:32:31 +08:00
parent 9d85ca07f4
commit 7f970d489f
51 changed files with 806 additions and 700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Oqtane.Server/Modules/SearchResults/Controllers/SearchResultsController.cs → Oqtane.Server/Controllers/SearchResultsController.cs
View File

@ -4,24 +4,22 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Oqtane.Controllers;
using Oqtane.Documentation;
using Oqtane.Enums;
using Oqtane.Infrastructure;
using Oqtane.Modules.SearchResults.Services;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Modules.SearchResults.Controllers
namespace Oqtane.Controllers
{
[Route(ControllerRoutes.ApiRoute)]
[PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")]
public class SearchResultsController : ModuleControllerBase
{
private readonly ISearchResultsService _searchResultsService;
private readonly ISearchService _searchService;
public SearchResultsController(ISearchResultsService searchResultsService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor)
public SearchResultsController(ISearchService searchService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor)
{
_searchResultsService = searchResultsService;
_searchService = searchService;
}
[HttpPost]
@ -30,9 +28,9 @@ namespace Oqtane.Modules.SearchResults.Controllers
{
try
{
return await _searchResultsService.SearchAsync(AuthEntityId(EntityNames.Module), searchQuery);
return await _searchService.SearchAsync(searchQuery);
}
catch(Exception ex)
catch (Exception ex)
{
_logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery);
HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest;

3
Oqtane.Server/Extensions/OqtaneServiceCollectionExtensions.cs
View File

@ -98,6 +98,7 @@ namespace Microsoft.Extensions.DependencyInjection
services.AddScoped<IUrlMappingService, UrlMappingService>();
services.AddScoped<IVisitorService, VisitorService>();
services.AddScoped<ISyncService, SyncService>();
services.AddScoped<ISearchResultsService, SearchResultsService>();
services.AddScoped<ISearchService, SearchService>();
services.AddScoped<ISearchProvider, DatabaseSearchProvider>();
@ -134,7 +135,7 @@ namespace Microsoft.Extensions.DependencyInjection
services.AddTransient<ILanguageRepository, LanguageRepository>();
services.AddTransient<IVisitorRepository, VisitorRepository>();
services.AddTransient<IUrlMappingRepository, UrlMappingRepository>();
services.AddTransient<ISearchDocumentRepository, SearchDocumentRepository>();
services.AddTransient<ISearchContentRepository, SearchContentRepository>();
// managers
services.AddTransient<IDBContextDependencies, DBContextDependencies>();

8
Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs
View File

@ -11,6 +11,8 @@ namespace Oqtane.Infrastructure
{
public class SearchIndexJob : HostedServiceBase
{
private const string SearchIndexStartTimeSettingName = "SearchIndex_StartTime";
public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory)
{
Name = "Search Index Job";
@ -54,7 +56,7 @@ namespace Oqtane.Infrastructure
private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository)
{
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName);
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName);
if(setting == null)
{
return null;
@ -65,14 +67,14 @@ namespace Oqtane.Infrastructure
private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository)
{
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName);
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, SearchIndexStartTimeSettingName);
if (setting == null)
{
setting = new Setting
{
EntityName = EntityNames.Site,
EntityId = siteId,
SettingName = Constants.SearchIndexStartTimeSettingName,
SettingName = SearchIndexStartTimeSettingName,
SettingValue = Convert.ToString(startTime),
};

4
Oqtane.Server/Infrastructure/SiteTemplates/DefaultSiteTemplate.cs
View File

@ -138,7 +138,7 @@ namespace Oqtane.SiteTemplates
Name = "Search Results",
Parent = "",
Order = 7,
Path = "search-results",
Path = "search",
Icon = "oi oi-magnifying-glass",
IsNavigation = false,
IsPersonalizable = false,
@ -148,7 +148,7 @@ namespace Oqtane.SiteTemplates
new Permission(PermissionNames.Edit, RoleNames.Admin, true)
},
PageTemplateModules = new List<PageTemplateModule> {
new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default,
new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default,
PermissionList = new List<Permission> {
new Permission(PermissionNames.View, RoleNames.Everyone, true),
new Permission(PermissionNames.View, RoleNames.Admin, true),

46
Oqtane.Server/Infrastructure/UpgradeManager.cs
View File

@ -66,6 +66,9 @@ namespace Oqtane.Infrastructure
case "5.1.0":
Upgrade_5_1_0(tenant, scope);
break;
case "5.2.0":
Upgrade_5_2_0(tenant, scope);
break;
}
}
}
@ -385,5 +388,48 @@ namespace Oqtane.Infrastructure
}
}
private void Upgrade_5_2_0(Tenant tenant, IServiceScope scope)
{
CreateSearchResultsPages(tenant, scope);
}
private void CreateSearchResultsPages(Tenant tenant, IServiceScope scope)
{
var pageTemplates = new List<PageTemplate>();
pageTemplates.Add(new PageTemplate
{
Name = "Search Results",
Parent = "",
Path = "search",
Icon = "oi oi-magnifying-glass",
IsNavigation = false,
IsPersonalizable = false,
PermissionList = new List<Permission> {
new Permission(PermissionNames.View, RoleNames.Everyone, true),
new Permission(PermissionNames.View, RoleNames.Admin, true),
new Permission(PermissionNames.Edit, RoleNames.Admin, true)
},
PageTemplateModules = new List<PageTemplateModule> {
new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.Admin.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default,
PermissionList = new List<Permission> {
new Permission(PermissionNames.View, RoleNames.Everyone, true),
new Permission(PermissionNames.View, RoleNames.Admin, true),
new Permission(PermissionNames.Edit, RoleNames.Admin, true)
}
}
}
});
var pages = scope.ServiceProvider.GetRequiredService<IPageRepository>();
var sites = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
foreach (var site in sites.GetSites().ToList())
{
if (!pages.GetPages(site.SiteId).ToList().Where(item => item.Path == "search").Any())
{
sites.CreatePages(site, pageTemplates, null);
}
}
}
}
}

85
Oqtane.Server/Managers/Search/ModuleSearchIndexManager.cs
View File

@ -12,6 +12,8 @@ namespace Oqtane.Managers.Search
{
public class ModuleSearchIndexManager : SearchIndexManagerBase
{
public const int ModuleSearchIndexManagerPriority = 200;
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<ModuleSearchIndexManager> _logger;
private readonly IPageModuleRepository _pageModuleRepostory;
@ -30,35 +32,41 @@ namespace Oqtane.Managers.Search
_pageRepository = pageRepository;
}
public override string Name => Constants.ModuleSearchIndexManagerName;
public override string Name => EntityNames.Module;
public override int Priority => Constants.ModuleSearchIndexManagerPriority;
public override int Priority => ModuleSearchIndexManagerPriority;
public override int IndexDocuments(int siteId, DateTime? startTime, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError)
public override int IndexContent(int siteId, DateTime? startTime, Action<IList<SearchContent>> processSearchContent, Action<string> handleError)
{
var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId);
var searchDocuments = new List<SearchDocument>();
var searchContentList = new List<SearchContent>();
foreach(var pageModule in pageModules)
{
var page = _pageRepository.GetPage(pageModule.PageId);
if(page == null || SearchUtils.IsSystemPage(page))
{
continue;
}
var module = pageModule.Module;
if (module.ModuleDefinition.ServerManagerType != "")
{
_logger.LogDebug($"Search: Begin index module {module.ModuleId}.");
var type = Type.GetType(module.ModuleDefinition.ServerManagerType);
if (type?.GetInterface("IModuleSearch") != null)
if (type?.GetInterface(nameof(ISearchable)) != null)
{
try
{
var moduleSearch = (IModuleSearch)ActivatorUtilities.CreateInstance(_serviceProvider, type);
var documents = moduleSearch.GetSearchDocuments(module, startTime.GetValueOrDefault(DateTime.MinValue));
if(documents != null)
var moduleSearch = (ISearchable)ActivatorUtilities.CreateInstance(_serviceProvider, type);
var contentList = moduleSearch.GetSearchContentList(module, startTime.GetValueOrDefault(DateTime.MinValue));
if(contentList != null)
{
foreach(var document in documents)
foreach(var searchContent in contentList)
{
SaveModuleMetaData(document, pageModule);
SaveModuleMetaData(searchContent, pageModule);
searchDocuments.Add(document);
searchContentList.Add(searchContent);
}
}
@ -73,54 +81,65 @@ namespace Oqtane.Managers.Search
}
}
processSearchDocuments(searchDocuments);
processSearchContent(searchContentList);
return searchDocuments.Count;
return searchContentList.Count;
}
private void SaveModuleMetaData(SearchDocument document, PageModule pageModule)
private void SaveModuleMetaData(SearchContent searchContent, PageModule pageModule)
{
document.EntryId = pageModule.ModuleId;
document.IndexerName = Name;
document.SiteId = pageModule.Module.SiteId;
document.LanguageCode = string.Empty;
searchContent.SiteId = pageModule.Module.SiteId;
if(document.ModifiedTime == DateTime.MinValue)
if(string.IsNullOrEmpty(searchContent.EntityName))
{
document.ModifiedTime = pageModule.ModifiedOn;
searchContent.EntityName = EntityNames.Module;
}
if (string.IsNullOrEmpty(document.AdditionalContent))
if(searchContent.EntityId == 0)
{
document.AdditionalContent = string.Empty;
searchContent.EntityId = pageModule.ModuleId;
}
if (searchContent.IsActive)
{
searchContent.IsActive = !pageModule.Module.IsDeleted;
}
if (searchContent.ModifiedTime == DateTime.MinValue)
{
searchContent.ModifiedTime = pageModule.ModifiedOn;
}
if (string.IsNullOrEmpty(searchContent.AdditionalContent))
{
searchContent.AdditionalContent = string.Empty;
}
var page = _pageRepository.GetPage(pageModule.PageId);
if (string.IsNullOrEmpty(document.Url) && page != null)
if (string.IsNullOrEmpty(searchContent.Url) && page != null)
{
document.Url = page.Url;
searchContent.Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
}
if (string.IsNullOrEmpty(document.Title) && page != null)
if (string.IsNullOrEmpty(searchContent.Title) && page != null)
{
document.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name;
searchContent.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name;
}
if (document.Properties == null)
if (searchContent.Properties == null)
{
document.Properties = new List<SearchDocumentProperty>();
searchContent.Properties = new List<SearchContentProperty>();
}
if(!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
if(!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() });
searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() });
}
if (!document.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName))
if (!searchContent.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() });
searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() });
}
}
}

17
Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs
View File

@ -11,7 +11,7 @@ namespace Oqtane.Managers.Search
{
public class ModuleSearchResultManager : ISearchResultManager
{
public string Name => Constants.ModuleSearchIndexManagerName;
public string Name => EntityNames.Module;
private readonly IServiceProvider _serviceProvider;
@ -37,26 +37,17 @@ namespace Oqtane.Managers.Search
return string.Empty;
}
public bool Visible(SearchDocument searchResult, SearchQuery searchQuery)
public bool Visible(SearchContent searchResult, SearchQuery searchQuery)
{
var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
var moduleIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchModuleIdPropertyName)?.Value ?? string.Empty;
if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)
&& !string.IsNullOrEmpty(moduleIdValue) && int.TryParse(moduleIdValue, out int moduleId))
if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
{
return CanViewPage(pageId, searchQuery.User) && CanViewModule(moduleId, searchQuery.User);
return CanViewPage(pageId, searchQuery.User);
}
return false;
}
private bool CanViewModule(int moduleId, User user)
{
var moduleRepository = _serviceProvider.GetRequiredService<IModuleRepository>();
var module = moduleRepository.GetModule(moduleId);
return module != null && !module.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, module.PermissionList);
}
private bool CanViewPage(int pageId, User user)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();

43
Oqtane.Server/Managers/Search/PageSearchIndexManager.cs
View File

@ -14,6 +14,8 @@ namespace Oqtane.Managers.Search
{
public class PageSearchIndexManager : SearchIndexManagerBase
{
private const int PageSearchIndexManagerPriority = 100;
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<ModuleSearchIndexManager> _logger;
private readonly IPageRepository _pageRepository;
@ -29,50 +31,50 @@ namespace Oqtane.Managers.Search
_pageRepository = pageRepository;
}
public override string Name => Constants.PageSearchIndexManagerName;
public override string Name => EntityNames.Page;
public override int Priority => Constants.PageSearchIndexManagerPriority;
public override int Priority => PageSearchIndexManagerPriority;
public override int IndexDocuments(int siteId, DateTime? startTime, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError)
public override int IndexContent(int siteId, DateTime? startTime, Action<IList<SearchContent>> processSearchContent, Action<string> handleError)
{
var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue);
var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue);
var searchDocuments = new List<SearchDocument>();
var searchContentList = new List<SearchContent>();
foreach(var page in pages)
{
try
{
if(IsSystemPage(page))
if(SearchUtils.IsSystemPage(page))
{
continue;
}
var document = new SearchDocument
var searchContent = new SearchContent
{
EntryId = page.PageId,
IndexerName = Name,
EntityName = EntityNames.Page,
EntityId = page.PageId,
SiteId = page.SiteId,
LanguageCode = string.Empty,
ModifiedTime = page.ModifiedOn,
AdditionalContent = string.Empty,
Url = page.Url ?? string.Empty,
Url = $"{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}",
Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name,
Description = string.Empty,
Body = $"{page.Name} {page.Title}"
Body = $"{page.Name} {page.Title}",
IsActive = !page.IsDeleted && Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate)
};
if (document.Properties == null)
if (searchContent.Properties == null)
{
document.Properties = new List<SearchDocumentProperty>();
searchContent.Properties = new List<SearchContentProperty>();
}
if (!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
if (!searchContent.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() });
searchContent.Properties.Add(new SearchContentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() });
}
searchDocuments.Add(document);
searchContentList.Add(searchContent);
}
catch(Exception ex)
{
@ -81,14 +83,9 @@ namespace Oqtane.Managers.Search
}
}
processSearchDocuments(searchDocuments);
processSearchContent(searchContentList);
return searchDocuments.Count;
}
private bool IsSystemPage(Models.Page page)
{
return page.Path.Contains("admin") || page.Path == "login" || page.Path == "register" || page.Path == "profile";
return searchContentList.Count;
}
}
}

46
Oqtane.Server/Managers/Search/PageSearchResultManager.cs
View File

@ -1,46 +0,0 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class PageSearchResultManager : ISearchResultManager
{
public string Name => Constants.PageSearchIndexManagerName;
private readonly IServiceProvider _serviceProvider;
public PageSearchResultManager(
IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public string GetUrl(SearchResult searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(searchResult.EntryId);
if (page != null)
{
return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
}
return string.Empty;
}
public bool Visible(SearchDocument searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(searchResult.EntryId);
return page != null && !page.IsDeleted
&& UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.View, page.PermissionList)
&& (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.Edit, page.PermissionList));
}
}
}

6
Oqtane.Server/Managers/Search/SearchIndexManagerBase.cs
View File

@ -10,6 +10,8 @@ namespace Oqtane.Managers.Search
{
public abstract class SearchIndexManagerBase : ISearchIndexManager
{
private const string SearchIndexManagerEnabledSettingFormat = "SearchIndexManager_{0}_Enabled";
private readonly IServiceProvider _serviceProvider;
public SearchIndexManagerBase(IServiceProvider serviceProvider)
@ -21,11 +23,11 @@ namespace Oqtane.Managers.Search
public abstract string Name { get; }
public abstract int IndexDocuments(int siteId, DateTime? startDate, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError);
public abstract int IndexContent(int siteId, DateTime? startDate, Action<IList<SearchContent>> processSearchContent, Action<string> handleError);
public virtual bool IsIndexEnabled(int siteId)
{
var settingName = string.Format(Constants.SearchIndexManagerEnabledSettingFormat, Name);
var settingName = string.Format(SearchIndexManagerEnabledSettingFormat, Name);
var settingRepository = _serviceProvider.GetRequiredService<ISettingRepository>();
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName);
return setting == null || setting.SettingValue == "true";

27
Oqtane.Server/Migrations/EntityBuilders/SearchDocumentEntityBuilder.cs → Oqtane.Server/Migrations/EntityBuilders/SearchContentEntityBuilder.cs
View File

@ -2,25 +2,26 @@ using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
using Oqtane.Models;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentEntityBuilder : AuditableBaseEntityBuilder<SearchDocumentEntityBuilder>
public class SearchContentEntityBuilder : AuditableBaseEntityBuilder<SearchContentEntityBuilder>
{
private const string _entityTableName = "SearchDocument";
private readonly PrimaryKey<SearchDocumentEntityBuilder> _primaryKey = new("PK_SearchDocument", x => x.SearchDocumentId);
private const string _entityTableName = "SearchContent";
private readonly PrimaryKey<SearchContentEntityBuilder> _primaryKey = new("PK_SearchContent", x => x.SearchContentId);
public SearchDocumentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
public SearchContentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
}
protected override SearchDocumentEntityBuilder BuildTable(ColumnsBuilder table)
protected override SearchContentEntityBuilder BuildTable(ColumnsBuilder table)
{
SearchDocumentId = AddAutoIncrementColumn(table, "SearchDocumentId");
EntryId = AddIntegerColumn(table, "EntryId");
IndexerName = AddStringColumn(table, "IndexerName", 50);
SearchContentId = AddAutoIncrementColumn(table, "SearchContentId");
EntityName = AddStringColumn(table, "EntityName", 50);
EntityId = AddIntegerColumn(table, "EntityId");
SiteId = AddIntegerColumn(table, "SiteId");
Title = AddStringColumn(table, "Title", 255);
Description = AddMaxStringColumn(table, "Description");
@ -29,18 +30,17 @@ namespace Oqtane.Migrations.EntityBuilders
ModifiedTime = AddDateTimeColumn(table, "ModifiedTime");
IsActive = AddBooleanColumn(table, "IsActive");
AdditionalContent = AddMaxStringColumn(table, "AdditionalContent");
LanguageCode = AddStringColumn(table, "LanguageCode", 20);
AddAuditableColumns(table);
return this;
}
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchContentId { get; private set; }
public OperationBuilder<AddColumnOperation> EntryId { get; private set; }
public OperationBuilder<AddColumnOperation> EntityName { get; private set; }
public OperationBuilder<AddColumnOperation> IndexerName { get; private set; }
public OperationBuilder<AddColumnOperation> EntityId { get; private set; }
public OperationBuilder<AddColumnOperation> SiteId { get; private set; }
@ -57,8 +57,5 @@ namespace Oqtane.Migrations.EntityBuilders
public OperationBuilder<AddColumnOperation> IsActive { get; private set; }
public OperationBuilder<AddColumnOperation> AdditionalContent { get; private set; }
public OperationBuilder<AddColumnOperation> LanguageCode { get; private set; }
}
}

40
Oqtane.Server/Migrations/EntityBuilders/SearchContentPropertyEntityBuilder.cs Normal file
View File

@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchContentPropertyEntityBuilder : BaseEntityBuilder<SearchContentPropertyEntityBuilder>
{
private const string _entityTableName = "SearchContentProperty";
private readonly PrimaryKey<SearchContentPropertyEntityBuilder> _primaryKey = new("PK_SearchContentProperty", x => x.PropertyId);
private readonly ForeignKey<SearchContentPropertyEntityBuilder> _searchContentForeignKey = new("FK_SearchContentProperty_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade);
public SearchContentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchContentForeignKey);
}
protected override SearchContentPropertyEntityBuilder BuildTable(ColumnsBuilder table)
{
PropertyId = AddAutoIncrementColumn(table, "PropertyId");
SearchContentId = AddIntegerColumn(table, "SearchContentId");
Name = AddStringColumn(table, "Name", 50);
Value = AddStringColumn(table, "Value", 50);
return this;
}
public OperationBuilder<AddColumnOperation> PropertyId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchContentId { get; private set; }
public OperationBuilder<AddColumnOperation> Name { get; private set; }
public OperationBuilder<AddColumnOperation> Value { get; private set; }
}
}

42
Oqtane.Server/Migrations/EntityBuilders/SearchContentWordCountEntityBuilder.cs Normal file
View File

@ -0,0 +1,42 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchContentWordsEntityBuilder : BaseEntityBuilder<SearchContentWordsEntityBuilder>
{
private const string _entityTableName = "SearchContentWords";
private readonly PrimaryKey<SearchContentWordsEntityBuilder> _primaryKey = new("PK_SearchContentWords", x => x.WordId);
private readonly ForeignKey<SearchContentWordsEntityBuilder> _searchContentForeignKey = new("FK_SearchContentWords_SearchContent", x => x.SearchContentId, "SearchContent", "SearchContentId", ReferentialAction.Cascade);
private readonly ForeignKey<SearchContentWordsEntityBuilder> _wordSourceForeignKey = new("FK_SearchContentWords_WordSource", x => x.WordSourceId, "SearchContentWordSource", "WordSourceId", ReferentialAction.Cascade);
public SearchContentWordsEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchContentForeignKey);
ForeignKeys.Add(_wordSourceForeignKey);
}
protected override SearchContentWordsEntityBuilder BuildTable(ColumnsBuilder table)
{
WordId = AddAutoIncrementColumn(table, "WordId");
SearchContentId = AddIntegerColumn(table, "SearchContentId");
WordSourceId = AddIntegerColumn(table, "WordSourceId");
Count = AddIntegerColumn(table, "Count");
return this;
}
public OperationBuilder<AddColumnOperation> WordId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchContentId { get; private set; }
public OperationBuilder<AddColumnOperation> WordSourceId { get; private set; }
public OperationBuilder<AddColumnOperation> Count { get; private set; }
}
}

31
Oqtane.Server/Migrations/EntityBuilders/SearchContentWordSourceEntityBuilder.cs Normal file
View File

@ -0,0 +1,31 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchContentWordSourceEntityBuilder : BaseEntityBuilder<SearchContentWordSourceEntityBuilder>
{
private const string _entityTableName = "SearchContentWordSource";
private readonly PrimaryKey<SearchContentWordSourceEntityBuilder> _primaryKey = new("PK_SearchContentWordSource", x => x.WordSourceId);
public SearchContentWordSourceEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
}
protected override SearchContentWordSourceEntityBuilder BuildTable(ColumnsBuilder table)
{
WordSourceId = AddAutoIncrementColumn(table, "WordSourceId");
Word = AddStringColumn(table, "Word", 255);
return this;
}
public OperationBuilder<AddColumnOperation> WordSourceId { get; private set; }
public OperationBuilder<AddColumnOperation> Word { get; private set; }
}
}

40
Oqtane.Server/Migrations/EntityBuilders/SearchDocumentPropertyEntityBuilder.cs
View File

@ -1,40 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentPropertyEntityBuilder : BaseEntityBuilder<SearchDocumentPropertyEntityBuilder>
{
private const string _entityTableName = "SearchDocumentProperty";
private readonly PrimaryKey<SearchDocumentPropertyEntityBuilder> _primaryKey = new("PK_SearchDocumentProperty", x => x.PropertyId);
private readonly ForeignKey<SearchDocumentPropertyEntityBuilder> _searchDocumentForeignKey = new("FK_SearchDocumentProperty_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade);
public SearchDocumentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchDocumentForeignKey);
}
protected override SearchDocumentPropertyEntityBuilder BuildTable(ColumnsBuilder table)
{
PropertyId = AddAutoIncrementColumn(table, "PropertyId");
SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId");
Name = AddStringColumn(table, "Name", 50);
Value = AddStringColumn(table, "Value", 50);
return this;
}
public OperationBuilder<AddColumnOperation> PropertyId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> Name { get; private set; }
public OperationBuilder<AddColumnOperation> Value { get; private set; }
}
}

37
Oqtane.Server/Migrations/EntityBuilders/SearchDocumentTagEntityBuilder.cs
View File

@ -1,37 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentTagEntityBuilder : BaseEntityBuilder<SearchDocumentTagEntityBuilder>
{
private const string _entityTableName = "SearchDocumentTag";
private readonly PrimaryKey<SearchDocumentTagEntityBuilder> _primaryKey = new("PK_SearchDocumentTag", x => x.TagId);
private readonly ForeignKey<SearchDocumentTagEntityBuilder> _searchDocumentForeignKey = new("FK_SearchDocumentTag_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade);
public SearchDocumentTagEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchDocumentForeignKey);
}
protected override SearchDocumentTagEntityBuilder BuildTable(ColumnsBuilder table)
{
TagId = AddAutoIncrementColumn(table, "TagId");
SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId");
Tag = AddStringColumn(table, "Tag", 50);
return this;
}
public OperationBuilder<AddColumnOperation> TagId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> Tag { get; private set; }
}
}

32
Oqtane.Server/Migrations/Tenant/05020001_AddSearchTables.cs
View File

@ -17,26 +17,34 @@ namespace Oqtane.Migrations.Tenant
protected override void Up(MigrationBuilder migrationBuilder)
{
var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentEntityBuilder.Create();
var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentEntityBuilder.Create();
var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentPropertyEntityBuilder.Create();
var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentPropertyEntityBuilder.Create();
var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentTagEntityBuilder.Create();
var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentWordSourceEntityBuilder.Create();
searchContentWordSourceEntityBuilder.AddIndex("IX_SearchContentWordSource", "Word", true);
var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentWordsEntityBuilder.Create();
}
protected override void Down(MigrationBuilder migrationBuilder)
{
var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentPropertyEntityBuilder.Drop();
var searchContentWordsEntityBuilder = new SearchContentWordsEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentWordsEntityBuilder.Drop();
var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentTagEntityBuilder.Drop();
var searchContentWordSourceEntityBuilder = new SearchContentWordSourceEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentWordSourceEntityBuilder.DropIndex("IX_SearchContentWordSource");
searchContentWordSourceEntityBuilder.Drop();
var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentEntityBuilder.Drop();
var searchContentPropertyEntityBuilder = new SearchContentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentPropertyEntityBuilder.Drop();
var searchContentEntityBuilder = new SearchContentEntityBuilder(migrationBuilder, ActiveDatabase);
searchContentEntityBuilder.Drop();
}
}
}

12
Oqtane.Server/Modules/HtmlText/Manager/HtmlTextManager.cs
View File

@ -17,7 +17,7 @@ using System;
namespace Oqtane.Modules.HtmlText.Manager
{
[PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")]
public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, IModuleSearch
public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, ISearchable
{
private readonly IServiceProvider _serviceProvider;
private readonly IHtmlTextRepository _htmlText;
@ -48,25 +48,25 @@ namespace Oqtane.Modules.HtmlText.Manager
return content;
}
public IList<SearchDocument> GetSearchDocuments(Module module, DateTime startDate)
public IList<SearchContent> GetSearchContentList(Module module, DateTime startDate)
{
var searchDocuments = new List<SearchDocument>();
var searchContentList = new List<SearchContent>();
var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId);
if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate))
{
var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First();
searchDocuments.Add(new SearchDocument
searchContentList.Add(new SearchContent
{
Title = module.Title,
Description = string.Empty,
Body = SearchUtils.Clean(htmltext.Content, true),
Body = htmltext.Content,
ModifiedTime = htmltext.ModifiedOn
});
}
return searchDocuments;
return searchContentList;
}
public void ImportModule(Module module, string content, string version)

36
Oqtane.Server/Modules/SearchResults/Services/SearchResultsService.cs
View File

@ -1,36 +0,0 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Oqtane.Documentation;
using Oqtane.Infrastructure;
using Oqtane.Models;
using Oqtane.Services;
namespace Oqtane.Modules.SearchResults.Services
{
[PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")]
public class ServerSearchResultsService : ISearchResultsService, ITransientService
{
private readonly ILogManager _logger;
private readonly IHttpContextAccessor _accessor;
private readonly Alias _alias;
private readonly ISearchService _searchService;
public ServerSearchResultsService(
ITenantManager tenantManager,
ILogManager logger,
IHttpContextAccessor accessor,
ISearchService searchService)
{
_logger = logger;
_accessor = accessor;
_alias = tenantManager.GetAlias();
_searchService = searchService;
}
public async Task<Models.SearchResults> SearchAsync(int moduleId, SearchQuery searchQuery)
{
var results = await _searchService.SearchAsync(searchQuery);
return results;
}
}
}

24
Oqtane.Server/Modules/SearchResults/Startup/ServerStartup.cs
View File

@ -1,24 +0,0 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Infrastructure;
using Oqtane.Modules.SearchResults.Services;
namespace Oqtane.Modules.SearchResults.Startup
{
public class ServerStartup : IServerStartup
{
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
}
public void ConfigureMvc(IMvcBuilder mvcBuilder)
{
}
public void ConfigureServices(IServiceCollection services)
{
services.AddTransient<ISearchResultsService, ServerSearchResultsService>();
}
}
}

1
Oqtane.Server/Oqtane.Server.csproj
View File

@ -33,6 +33,7 @@
<EmbeddedResource Include="Scripts\MigrateTenant.sql" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.5" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.5" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />

165
Oqtane.Server/Providers/DatabaseSearchProvider.cs
View File

@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using HtmlAgilityPack;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Services;
@ -13,27 +16,28 @@ namespace Oqtane.Providers
{
public class DatabaseSearchProvider : ISearchProvider
{
private readonly ISearchDocumentRepository _searchDocumentRepository;
private readonly ISearchContentRepository _searchContentRepository;
private const float TitleBoost = 100f;
private const float DescriptionBoost = 10f;
private const float BodyBoost = 10f;
private const float AdditionalContentBoost = 5f;
private const string IgnoreWords = "the,be,to,of,and,a,i,in,that,have,it,for,not,on,with,he,as,you,do,at,this,but,his,by,from,they,we,say,her,she,or,an,will,my,one,all,would,there,their,what,so,up,out,if,about,who,get,which,go,me,when,make,can,like,time,no,just,him,know,take,people,into,year,your,good,some,could,them,see,other,than,then,now,look,only,come,its,over,think,also,back,after,use,two,how,our,work,first,well,way,even,new,want,because,any,these,give,day,most,us";
private const int WordMinLength = 3;
public string Name => Constants.DefaultSearchProviderName;
public DatabaseSearchProvider(ISearchDocumentRepository searchDocumentRepository)
public DatabaseSearchProvider(ISearchContentRepository searchContentRepository)
{
_searchDocumentRepository = searchDocumentRepository;
_searchContentRepository = searchContentRepository;
}
public void Commit()
{
}
public void DeleteDocument(string id)
public void DeleteSearchContent(string id)
{
_searchDocumentRepository.DeleteSearchDocument(id);
_searchContentRepository.DeleteSearchContent(id);
}
public bool Optimize()
@ -43,25 +47,28 @@ namespace Oqtane.Providers
public void ResetIndex()
{
_searchDocumentRepository.DeleteAllSearchDocuments();
_searchContentRepository.DeleteAllSearchContent();
}
public void SaveDocument(SearchDocument document, bool autoCommit = false)
public void SaveSearchContent(SearchContent searchContent, bool autoCommit = false)
{
//remove exist document
_searchDocumentRepository.DeleteSearchDocument(document.IndexerName, document.EntryId);
_searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId);
_searchDocumentRepository.AddSearchDocument(document);
_searchContentRepository.AddSearchContent(searchContent);
//save the index words
AnalyzeSearchContent(searchContent);
}
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchDocument, SearchQuery, bool> validateFunc)
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchContent, SearchQuery, bool> validateFunc)
{
var totalResults = 0;
var documents = await _searchDocumentRepository.GetSearchDocumentsAsync(searchQuery);
var searchContentList = await _searchContentRepository.GetSearchContentListAsync(searchQuery);
//convert the search documents to search results.
var results = documents
//convert the search content to search results.
var results = searchContentList
.Where(i => validateFunc(i, searchQuery))
.Select(i => ConvertToSearchResult(i, searchQuery));
@ -99,7 +106,7 @@ namespace Oqtane.Providers
//remove duplicated results based on page id for Page and Module types
results = results.DistinctBy(i =>
{
if (i.IndexerName == Constants.PageSearchIndexManagerName || i.IndexerName == Constants.ModuleSearchIndexManagerName)
if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module)
{
var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
@ -119,45 +126,44 @@ namespace Oqtane.Providers
};
}
private SearchResult ConvertToSearchResult(SearchDocument searchDocument, SearchQuery searchQuery)
private SearchResult ConvertToSearchResult(SearchContent searchContent, SearchQuery searchQuery)
{
var searchResult = new SearchResult()
{
SearchDocumentId = searchDocument.SearchDocumentId,
SiteId = searchDocument.SiteId,
IndexerName = searchDocument.IndexerName,
EntryId = searchDocument.EntryId,
Title = searchDocument.Title,
Description = searchDocument.Description,
Body = searchDocument.Body,
Url = searchDocument.Url,
ModifiedTime = searchDocument.ModifiedTime,
Tags = searchDocument.Tags,
Properties = searchDocument.Properties,
Snippet = BuildSnippet(searchDocument, searchQuery),
Score = CalculateScore(searchDocument, searchQuery)
SearchContentId = searchContent.SearchContentId,
SiteId = searchContent.SiteId,
EntityName = searchContent.EntityName,
EntityId = searchContent.EntityId,
Title = searchContent.Title,
Description = searchContent.Description,
Body = searchContent.Body,
Url = searchContent.Url,
ModifiedTime = searchContent.ModifiedTime,
Properties = searchContent.Properties,
Snippet = BuildSnippet(searchContent, searchQuery),
Score = CalculateScore(searchContent, searchQuery)
};
return searchResult;
}
private float CalculateScore(SearchDocument searchDocument, SearchQuery searchQuery)
private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery)
{
var score = 0f;
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
score += Regex.Matches(searchDocument.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost;
score += Regex.Matches(searchDocument.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost;
score += Regex.Matches(searchDocument.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost;
score += Regex.Matches(searchDocument.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost;
score += Regex.Matches(searchContent.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost;
score += Regex.Matches(searchContent.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost;
score += Regex.Matches(searchContent.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost;
score += Regex.Matches(searchContent.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost;
}
return score / 100;
}
private string BuildSnippet(SearchDocument searchDocument, SearchQuery searchQuery)
private string BuildSnippet(SearchContent searchContent, SearchQuery searchQuery)
{
var content = $"{searchDocument.Title} {searchDocument.Description} {searchDocument.Body}";
var content = $"{searchContent.Title} {searchContent.Description} {searchContent.Body}";
var snippet = string.Empty;
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
@ -198,5 +204,92 @@ namespace Oqtane.Providers
return snippet;
}
private void AnalyzeSearchContent(SearchContent searchContent)
{
//analyze the search content and save the index words
var indexContent = $"{searchContent.Title} {searchContent.Description} {searchContent.Body} {searchContent.AdditionalContent}";
var words = GetWords(indexContent, WordMinLength);
var existWords = _searchContentRepository.GetWords(searchContent.SearchContentId);
foreach (var kvp in words)
{
var word = existWords.FirstOrDefault(i => i.WordSource.Word == kvp.Key);
if (word != null)
{
word.Count = kvp.Value;
_searchContentRepository.UpdateSearchContentWords(word);
}
else
{
var wordSource = _searchContentRepository.GetSearchContentWordSource(kvp.Key);
if (wordSource == null)
{
wordSource = _searchContentRepository.AddSearchContentWordSource(new SearchContentWordSource { Word = kvp.Key });
}
word = new SearchContentWords
{
SearchContentId = searchContent.SearchContentId,
WordSourceId = wordSource.WordSourceId,
Count = kvp.Value
};
_searchContentRepository.AddSearchContentWords(word);
}
}
}
private static Dictionary<string, int> GetWords(string content, int minLength)
{
content = WebUtility.HtmlDecode(content);
var words = new Dictionary<string, int>();
var ignoreWords = IgnoreWords.Split(',');
var page = new HtmlDocument();
page.LoadHtml(content);
var phrases = page.DocumentNode.Descendants().Where(i =>
i.NodeType == HtmlNodeType.Text &&
i.ParentNode.Name != "script" &&
i.ParentNode.Name != "style" &&
!string.IsNullOrEmpty(i.InnerText.Trim())
).Select(i => FormatText(i.InnerText));
foreach (var phrase in phrases)
{
if (!string.IsNullOrEmpty(phrase))
{
foreach (var word in phrase.Split(' '))
{
if (word.Length >= minLength && !ignoreWords.Contains(word))
{
if (!words.ContainsKey(word))
{
words.Add(word, 1);
}
else
{
words[word] += 1;
}
}
}
}
}
return words;
}
private static string FormatText(string text)
{
text = HtmlEntity.DeEntitize(text);
foreach (var punctuation in ".?!,;:-_()[]{}'\"/\\".ToCharArray())
{
text = text.Replace(punctuation, ' ');
}
text = text.Replace(" ", " ").ToLower().Trim();
return text;
}
}
}

7
Oqtane.Server/Repository/Context/TenantDBContext.cs
View File

@ -29,8 +29,9 @@ namespace Oqtane.Repository
public virtual DbSet<Language> Language { get; set; }
public virtual DbSet<Visitor> Visitor { get; set; }
public virtual DbSet<UrlMapping> UrlMapping { get; set; }
public virtual DbSet<SearchDocument> SearchDocument { get; set; }
public virtual DbSet<SearchDocumentProperty> SearchDocumentProperty { get; set; }
public virtual DbSet<SearchDocumentTag> SearchDocumentTag { get; set; }
public virtual DbSet<SearchContent> SearchContent { get; set; }
public virtual DbSet<SearchContentProperty> SearchContentProperty { get; set; }
public virtual DbSet<SearchContentWords> SearchContentWords { get; set; }
public virtual DbSet<SearchContentWordSource> SearchContentWordSource { get; set; }
}
}

24
Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Oqtane.Models;
namespace Oqtane.Repository
{
public interface ISearchContentRepository
{
Task<IEnumerable<SearchContent>> GetSearchContentListAsync(SearchQuery searchQuery);
SearchContent AddSearchContent(SearchContent searchContent);
void DeleteSearchContent(int searchContentId);
void DeleteSearchContent(string entityName, int entryId);
void DeleteSearchContent(string uniqueKey);
void DeleteAllSearchContent();
SearchContentWordSource GetSearchContentWordSource(string word);
SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource);
IEnumerable<SearchContentWords> GetWords(int searchContentId);
SearchContentWords AddSearchContentWords(SearchContentWords word);
SearchContentWords UpdateSearchContentWords(SearchContentWords word);
}
}

17
Oqtane.Server/Repository/Interfaces/ISearchDocumentRepository.cs
View File

@ -1,17 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Oqtane.Models;
namespace Oqtane.Repository
{
public interface ISearchDocumentRepository
{
Task<IEnumerable<SearchDocument>> GetSearchDocumentsAsync(SearchQuery searchQuery);
SearchDocument AddSearchDocument(SearchDocument searchDocument);
void DeleteSearchDocument(int searchDocumentId);
void DeleteSearchDocument(string indexerName, int entryId);
void DeleteSearchDocument(string uniqueKey);
void DeleteAllSearchDocuments();
}
}

167
Oqtane.Server/Repository/SearchContentRepository.cs Normal file
View File

@ -0,0 +1,167 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Oqtane.Models;
using Oqtane.Shared;
namespace Oqtane.Repository
{
public class SearchContentRepository : ISearchContentRepository
{
private readonly IDbContextFactory<TenantDBContext> _dbContextFactory;
public SearchContentRepository(IDbContextFactory<TenantDBContext> dbContextFactory)
{
_dbContextFactory = dbContextFactory;
}
public async Task<IEnumerable<SearchContent>> GetSearchContentListAsync(SearchQuery searchQuery)
{
using var db = _dbContextFactory.CreateDbContext();
var searchContentList = db.SearchContent.AsNoTracking()
.Include(i => i.Properties)
.Where(i => i.SiteId == searchQuery.SiteId && i.IsActive);
if (searchQuery.EntityNames != null && searchQuery.EntityNames.Any())
{
searchContentList = searchContentList.Where(i => searchQuery.EntityNames.Contains(i.EntityName));
}
if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue)
{
searchContentList = searchContentList.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc);
}
if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue)
{
searchContentList = searchContentList.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc);
}
if (searchQuery.Properties != null && searchQuery.Properties.Any())
{
foreach (var property in searchQuery.Properties)
{
searchContentList = searchContentList.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value));
}
}
var filteredContentList = new List<SearchContent>();
if (!string.IsNullOrEmpty(searchQuery.Keywords))
{
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
filteredContentList.AddRange(await searchContentList.Where(i => i.Words.Any(w => w.WordSource.Word.StartsWith(keyword))).ToListAsync());
}
}
return filteredContentList.DistinctBy(i => i.UniqueKey);
}
public SearchContent AddSearchContent(SearchContent searchContent)
{
using var context = _dbContextFactory.CreateDbContext();
context.SearchContent.Add(searchContent);
if(searchContent.Properties != null && searchContent.Properties.Any())
{
foreach(var property in searchContent.Properties)
{
property.SearchContentId = searchContent.SearchContentId;
context.SearchContentProperty.Add(property);
}
}
context.SaveChanges();
return searchContent;
}
public void DeleteSearchContent(int searchContentId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchContent = db.SearchContent.Find(searchContentId);
db.SearchContent.Remove(searchContent);
db.SaveChanges();
}
public void DeleteSearchContent(string entityName, int entryId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchContent = db.SearchContent.FirstOrDefault(i => i.EntityName == entityName && i.EntityId == entryId);
if(searchContent != null)
{
db.SearchContent.Remove(searchContent);
db.SaveChanges();
}
}
public void DeleteSearchContent(string uniqueKey)
{
using var db = _dbContextFactory.CreateDbContext();
var searchContent = db.SearchContent.FirstOrDefault(i => (i.EntityName + ":" + i.EntityId) == uniqueKey);
if (searchContent != null)
{
db.SearchContent.Remove(searchContent);
db.SaveChanges();
}
}
public void DeleteAllSearchContent()
{
using var db = _dbContextFactory.CreateDbContext();
db.SearchContent.RemoveRange(db.SearchContent);
db.SaveChanges();
}
public SearchContentWordSource GetSearchContentWordSource(string word)
{
if(string.IsNullOrEmpty(word))
{
return null;
}
using var db = _dbContextFactory.CreateDbContext();
return db.SearchContentWordSource.FirstOrDefault(i => i.Word == word);
}
public SearchContentWordSource AddSearchContentWordSource(SearchContentWordSource wordSource)
{
using var db = _dbContextFactory.CreateDbContext();
db.SearchContentWordSource.Add(wordSource);
db.SaveChanges();
return wordSource;
}
public IEnumerable<SearchContentWords> GetWords(int searchContentId)
{
using var db = _dbContextFactory.CreateDbContext();
return db.SearchContentWords
.Include(i => i.WordSource)
.Where(i => i.SearchContentId == searchContentId).ToList();
}
public SearchContentWords AddSearchContentWords(SearchContentWords word)
{
using var db = _dbContextFactory.CreateDbContext();
db.SearchContentWords.Add(word);
db.SaveChanges();
return word;
}
public SearchContentWords UpdateSearchContentWords(SearchContentWords word)
{
using var db = _dbContextFactory.CreateDbContext();
db.Entry(word).State = EntityState.Modified;
db.SaveChanges();
return word;
}
}
}

136
Oqtane.Server/Repository/SearchDocumentRepository.cs
View File

@ -1,136 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Oqtane.Models;
using Oqtane.Shared;
namespace Oqtane.Repository
{
public class SearchDocumentRepository : ISearchDocumentRepository
{
private readonly IDbContextFactory<TenantDBContext> _dbContextFactory;
public SearchDocumentRepository(IDbContextFactory<TenantDBContext> dbContextFactory)
{
_dbContextFactory = dbContextFactory;
}
public async Task<IEnumerable<SearchDocument>> GetSearchDocumentsAsync(SearchQuery searchQuery)
{
using var db = _dbContextFactory.CreateDbContext();
var documents = db.SearchDocument.AsNoTracking()
.Include(i => i.Properties)
.Include(i => i.Tags)
.Where(i => i.SiteId == searchQuery.SiteId);
if (searchQuery.Sources != null && searchQuery.Sources.Any())
{
documents = documents.Where(i => searchQuery.Sources.Contains(i.IndexerName));
}
if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue)
{
documents = documents.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc);
}
if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue)
{
documents = documents.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc);
}
if (searchQuery.Tags != null && searchQuery.Tags.Any())
{
foreach (var tag in searchQuery.Tags)
{
documents = documents.Where(i => i.Tags.Any(t => t.Tag == tag));
}
}
if (searchQuery.Properties != null && searchQuery.Properties.Any())
{
foreach (var property in searchQuery.Properties)
{
documents = documents.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value));
}
}
var filteredDocuments = new List<SearchDocument>();
if (!string.IsNullOrEmpty(searchQuery.Keywords))
{
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
filteredDocuments.AddRange(await documents.Where(i => i.Title.Contains(keyword) || i.Description.Contains(keyword) || i.Body.Contains(keyword)).ToListAsync());
}
}
return filteredDocuments.DistinctBy(i => i.UniqueKey);
}
public SearchDocument AddSearchDocument(SearchDocument searchDocument)
{
using var context = _dbContextFactory.CreateDbContext();
context.SearchDocument.Add(searchDocument);
if(searchDocument.Properties != null && searchDocument.Properties.Any())
{
foreach(var property in searchDocument.Properties)
{
property.SearchDocumentId = searchDocument.SearchDocumentId;
context.SearchDocumentProperty.Add(property);
}
}
if (searchDocument.Tags != null && searchDocument.Tags.Any())
{
foreach (var tag in searchDocument.Tags)
{
tag.SearchDocumentId = searchDocument.SearchDocumentId;
context.SearchDocumentTag.Add(tag);
}
}
context.SaveChanges();
return searchDocument;
}
public void DeleteSearchDocument(int searchDocumentId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.Find(searchDocumentId);
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
public void DeleteSearchDocument(string indexerName, int entryId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.FirstOrDefault(i => i.IndexerName == indexerName && i.EntryId == entryId);
if(searchDocument != null)
{
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
}
public void DeleteSearchDocument(string uniqueKey)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.FirstOrDefault(i => (i.IndexerName + ":" + i.EntryId) == uniqueKey);
if (searchDocument != null)
{
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
}
public void DeleteAllSearchDocuments()
{
using var db = _dbContextFactory.CreateDbContext();
db.SearchDocument.RemoveRange(db.SearchDocument);
db.SaveChanges();
}
}
}

81
Oqtane.Server/Services/SearchService.cs
View File

@ -1,23 +1,30 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Oqtane.Infrastructure;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Shared;
namespace Oqtane.Services
{
public class SearchService : ISearchService
{
private const string SearchProviderSettingName = "SearchProvider";
private const string SearchEnabledSettingName = "SearchEnabled";
private readonly IServiceProvider _serviceProvider;
private readonly ITenantManager _tenantManager;
private readonly IAliasRepository _aliasRepository;
private readonly ISettingRepository _settingRepository;
private readonly IPermissionRepository _permissionRepository;
private readonly ILogger<SearchService> _logger;
private readonly IMemoryCache _cache;
@ -26,12 +33,14 @@ namespace Oqtane.Services
ITenantManager tenantManager,
IAliasRepository aliasRepository,
ISettingRepository settingRepository,
IPermissionRepository permissionRepository,
ILogger<SearchService> logger,
IMemoryCache cache)
{
_tenantManager = tenantManager;
_aliasRepository = aliasRepository;
_settingRepository = settingRepository;
_permissionRepository = permissionRepository;
_serviceProvider = serviceProvider;
_logger = logger;
_cache = cache;
@ -68,8 +77,8 @@ namespace Oqtane.Services
{
_logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}");
var count = searchIndexManager.IndexDocuments(siteId, startTime, SaveIndexDocuments, handleError);
logNote($"Search: Indexer {searchIndexManager.Name} processed {count} documents.<br />");
var count = searchIndexManager.IndexContent(siteId, startTime, SaveSearchContent, handleError);
logNote($"Search: Indexer {searchIndexManager.Name} processed {count} search content.<br />");
_logger.LogDebug($"Search: End Index {searchIndexManager.Name}");
}
@ -79,7 +88,7 @@ namespace Oqtane.Services
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery)
{
var searchProvider = GetSearchProvider(searchQuery.SiteId);
var searchResults = await searchProvider.SearchAsync(searchQuery, HasViewPermission);
var searchResults = await searchProvider.SearchAsync(searchQuery, Visible);
//generate the document url if it's not set.
foreach (var result in searchResults.Results)
@ -108,7 +117,7 @@ namespace Oqtane.Services
private string GetSearchProviderSetting(int siteId)
{
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchProviderSettingName);
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchProviderSettingName);
if(!string.IsNullOrEmpty(setting?.SettingValue))
{
return setting.SettingValue;
@ -119,7 +128,7 @@ namespace Oqtane.Services
private bool SearchEnabled(int siteId)
{
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchEnabledSettingName);
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchEnabledSettingName);
if (!string.IsNullOrEmpty(setting?.SettingValue))
{
return bool.TryParse(setting.SettingValue, out bool enabled) && enabled;
@ -167,21 +176,22 @@ namespace Oqtane.Services
return managers.ToList();
}
private void SaveIndexDocuments(IList<SearchDocument> searchDocuments)
private void SaveSearchContent(IList<SearchContent> searchContentList)
{
if(searchDocuments.Any())
if(searchContentList.Any())
{
var searchProvider = GetSearchProvider(searchDocuments.First().SiteId);
var searchProvider = GetSearchProvider(searchContentList.First().SiteId);
foreach (var searchDocument in searchDocuments)
foreach (var searchContent in searchContentList)
{
try
{
searchProvider.SaveDocument(searchDocument);
CleanSearchContent(searchContent);
searchProvider.SaveSearchContent(searchContent);
}
catch(Exception ex)
{
_logger.LogError(ex, $"Search: Save search document {searchDocument.UniqueKey} failed.");
_logger.LogError(ex, $"Search: Save search content {searchContent.UniqueKey} failed.");
}
}
@ -190,19 +200,30 @@ namespace Oqtane.Services
}
}
private bool HasViewPermission(SearchDocument searchDocument, SearchQuery searchQuery)
private bool Visible(SearchContent searchContent, SearchQuery searchQuery)
{
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchDocument.IndexerName);
if(!HasViewPermission(searchQuery.SiteId, searchQuery.User, searchContent.EntityName, searchContent.EntityId))
{
return false;
}
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchContent.EntityName);
if (searchResultManager != null)
{
return searchResultManager.Visible(searchDocument, searchQuery);
return searchResultManager.Visible(searchContent, searchQuery);
}
return true;
}
private bool HasViewPermission(int siteId, User user, string entityName, int entityId)
{
var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList();
return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions);
}
private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery)
{
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.IndexerName);
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.EntityName);
if(searchResultManager != null)
{
return searchResultManager.GetUrl(result, searchQuery);
@ -210,5 +231,35 @@ namespace Oqtane.Services
return string.Empty;
}
private void CleanSearchContent(SearchContent searchContent)
{
searchContent.Title = GetCleanContent(searchContent.Title);
searchContent.Description = GetCleanContent(searchContent.Description);
searchContent.Body = GetCleanContent(searchContent.Body);
searchContent.AdditionalContent = GetCleanContent(searchContent.AdditionalContent);
}
private string GetCleanContent(string content)
{
if(string.IsNullOrWhiteSpace(content))
{
return string.Empty;
}
content = WebUtility.HtmlDecode(content);
var page = new HtmlDocument();
page.LoadHtml(content);
var phrases = page.DocumentNode.Descendants().Where(i =>
i.NodeType == HtmlNodeType.Text &&
i.ParentNode.Name != "script" &&
i.ParentNode.Name != "style" &&
!string.IsNullOrEmpty(i.InnerText.Trim())
).Select(i => i.InnerText);
return string.Join(" ", phrases);
}
}
}

0
Oqtane.Server/wwwroot/Modules/Oqtane.Modules.SearchResults/Module.css → Oqtane.Server/wwwroot/Modules/Oqtane.Modules.Admin.SearchResults/Module.css
View File