diff --git a/Oqtane.Client/Modules/Admin/Visitors/Index.razor b/Oqtane.Client/Modules/Admin/Visitors/Index.razor
index ea5510c0..1cb46d3e 100644
--- a/Oqtane.Client/Modules/Admin/Visitors/Index.razor
+++ b/Oqtane.Client/Modules/Admin/Visitors/Index.razor
@@ -100,6 +100,18 @@ else
+
+
+
+
+
}
@@ -113,6 +125,7 @@ else
private string _filter = "";
private int _retention = 30;
private string _correlation = "true";
+ private string _robots = "";
public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Admin;
@@ -139,7 +152,8 @@ else
_filter = SettingService.GetSetting(settings, "VisitorFilter", Constants.DefaultVisitorFilter);
_retention = int.Parse(SettingService.GetSetting(settings, "VisitorRetention", "30"));
_correlation = SettingService.GetSetting(settings, "VisitorCorrelation", "true");
- }
+ _robots = SettingService.GetSetting(settings, "Robots", "");
+ }
private async void TypeChanged(ChangeEventArgs e)
{
@@ -191,6 +205,7 @@ else
settings = SettingService.SetSetting(settings, "VisitorFilter", _filter, true);
settings = SettingService.SetSetting(settings, "VisitorRetention", _retention.ToString(), true);
settings = SettingService.SetSetting(settings, "VisitorCorrelation", _correlation, true);
+ settings = SettingService.SetSetting(settings, "Robots", _robots, true);
await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
diff --git a/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx b/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
index 37599ccb..a6dab428 100644
--- a/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
+++ b/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
@@ -198,4 +198,13 @@
Session Duration:
+
+ Robots.txt
+
+
+ Instructions:
+
+
+ Specify your robots.txt instructions to provide bots with guidance on which parts of your site should be indexed
+
\ No newline at end of file
diff --git a/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs b/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
index 4af3d6c2..5afc1326 100644
--- a/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
+++ b/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
@@ -74,8 +74,21 @@ namespace Oqtane.Infrastructure
// handle robots.txt root request (does not support subfolder aliases)
if (context.Request.Path.StartsWithSegments("/robots.txt") && string.IsNullOrEmpty(alias.Path))
{
- // allow all user agents and specify site map
- var robots = $"User-agent: *\n\nSitemap: {context.Request.Scheme}://{alias.Name}/sitemap.xml";
+ string robots = "";
+ if (sitesettings.ContainsKey("Robots") && !string.IsNullOrEmpty(sitesettings["Robots"]))
+ {
+ robots = sitesettings["Robots"];
+ }
+ else
+ {
+ // allow all user agents by default
+ robots = $"User-agent: *";
+ }
+ if (!robots.ToLower().Contains("sitemap"))
+ {
+ // add sitemap if not specified
+ robots += $"\n\nSitemap: {context.Request.Scheme}://{alias.Name}/sitemap.xml";
+ }
context.Response.ContentType = "text/plain";
await context.Response.WriteAsync(robots);
return;