From 020b7233d05d3cbba11c41657e22ddb6d3229782 Mon Sep 17 00:00:00 2001
From: sbwalker <shaun.walker@siliqon.com>
Date: Tue, 8 Apr 2025 09:23:22 -0400
Subject: [PATCH] fix #5223 - allow robots.txt to be customized for each site

---
 .../Modules/Admin/Visitors/Index.razor          | 17 ++++++++++++++++-
 .../Resources/Modules/Admin/Visitors/Index.resx |  9 +++++++++
 .../Middleware/TenantMiddleware.cs              | 17 +++++++++++++++--
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/Oqtane.Client/Modules/Admin/Visitors/Index.razor b/Oqtane.Client/Modules/Admin/Visitors/Index.razor
index ea5510c0..1cb46d3e 100644
--- a/Oqtane.Client/Modules/Admin/Visitors/Index.razor
+++ b/Oqtane.Client/Modules/Admin/Visitors/Index.razor
@@ -100,6 +100,18 @@ else
 			<br />
 			<button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
 		</TabPanel>
+        <TabPanel Name="Robots" Heading="Robots.txt" ResourceKey="Robots">
+			<div class="container">
+				<div class="row mb-1 align-items-center">
+                    <Label Class="col-sm-3" For="robots" HelpText="Specify your robots.txt instructions to provide bots with guidance on which parts of your site should be indexed" ResourceKey="Robots">Instructions: </Label>
+					<div class="col-sm-9">
+                        <textarea id="robots" class="form-control" @bind="@_robots" rows="3"></textarea>
+                    </div>
+				</div>
+            </div>
+            <br />
+            <button type="button" class="btn btn-success" @onclick="SaveSiteSettings">@SharedLocalizer["Save"]</button>
+        </TabPanel>
 	</TabStrip>
 }
 
@@ -113,6 +125,7 @@ else
     private string _filter = "";
 	private int _retention = 30;
 	private string _correlation = "true";
+    private string _robots = "";
 
 	public override SecurityAccessLevel SecurityAccessLevel => SecurityAccessLevel.Admin;
 
@@ -139,7 +152,8 @@ else
         _filter = SettingService.GetSetting(settings, "VisitorFilter", Constants.DefaultVisitorFilter);
 		_retention = int.Parse(SettingService.GetSetting(settings, "VisitorRetention", "30"));
 		_correlation = SettingService.GetSetting(settings, "VisitorCorrelation", "true");
-	}
+        _robots = SettingService.GetSetting(settings, "Robots", "");
+    }
 
 	private async void TypeChanged(ChangeEventArgs e)
 	{
@@ -191,6 +205,7 @@ else
             settings = SettingService.SetSetting(settings, "VisitorFilter", _filter, true);
 			settings = SettingService.SetSetting(settings, "VisitorRetention", _retention.ToString(), true);
 			settings = SettingService.SetSetting(settings, "VisitorCorrelation", _correlation, true);
+            settings = SettingService.SetSetting(settings, "Robots", _robots, true);
             await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
 
 			AddModuleMessage(Localizer["Success.SaveSiteSettings"], MessageType.Success);
diff --git a/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx b/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
index 37599ccb..a6dab428 100644
--- a/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
+++ b/Oqtane.Client/Resources/Modules/Admin/Visitors/Index.resx
@@ -198,4 +198,13 @@
   <data name="Duration.Text" xml:space="preserve">
     <value>Session Duration:</value>
   </data>
+  <data name="Robots.Heading" xml:space="preserve">
+    <value>Robots.txt</value>
+  </data>
+  <data name="Robots.Text" xml:space="preserve">
+    <value>Instructions:</value>
+  </data>
+  <data name="Robots.HelpText" xml:space="preserve">
+    <value>Specify your robots.txt instructions to provide bots with guidance on which parts of your site should be indexed</value>
+  </data>
 </root>
\ No newline at end of file
diff --git a/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs b/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
index 4af3d6c2..5afc1326 100644
--- a/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
+++ b/Oqtane.Server/Infrastructure/Middleware/TenantMiddleware.cs
@@ -74,8 +74,21 @@ namespace Oqtane.Infrastructure
                     // handle robots.txt root request (does not support subfolder aliases)
                     if (context.Request.Path.StartsWithSegments("/robots.txt") && string.IsNullOrEmpty(alias.Path))
                     {
-                        // allow all user agents and specify site map
-                        var robots = $"User-agent: *\n\nSitemap: {context.Request.Scheme}://{alias.Name}/sitemap.xml";
+                        string robots = "";
+                        if (sitesettings.ContainsKey("Robots") && !string.IsNullOrEmpty(sitesettings["Robots"]))
+                        {
+                            robots = sitesettings["Robots"];
+                        }
+                        else
+                        {
+                            // allow all user agents by default
+                            robots = $"User-agent: *";
+                        }
+                        if (!robots.ToLower().Contains("sitemap"))
+                        {
+                            // add sitemap if not specified
+                            robots += $"\n\nSitemap: {context.Request.Scheme}://{alias.Name}/sitemap.xml";
+                        }
                         context.Response.ContentType = "text/plain";
                         await context.Response.WriteAsync(robots);
                         return;