upgrade step to add robots.txt rule for googlebot

src/redturtle/volto/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <metadata>
-  <version>4200</version>
+  <version>4300</version>
   <dependencies>
     <dependency>profile-plone.volto:default</dependency>
     <dependency>profile-plone.app.caching:with-caching-proxy</dependency>

src/redturtle/volto/upgrades.py

@@ -464,3 +464,47 @@ def to_4200(context):
     logger.info("Add redturtle.volto controlpanel")
     update_registry(context)
     update_controlpanel(context)
+
+
+def upgrade_robots_txt(context):
+    robots = api.portal.get_registry_record('plone.robots_txt')
+    lines = robots.splitlines()
+
+    googlebot_user_agent = 'User-Agent: Googlebot'
+    # I saw this writed also as googlebot, so:
+    normalized_google_bot_user_agent = googlebot_user_agent.lower().replace(' ','')
+
+    useragent_string = 'User-Agent: '
+    normalized_useragent_string = useragent_string.lower().replace(' ', '')
+
+    googlebot_allow_rule = 'Allow: /*?expand*'
+
+    googlebot_index = -1
+    allow_rule_present = False
+    for i, line in enumerate(lines):
+        if line.lower().replace(' ','') == normalized_google_bot_user_agent:
+            googlebot_index = i
+        elif (googlebot_index != -1 and
+            line.strip().lower() == googlebot_allow_rule.lower()):
+            allow_rule_present = True
+
+    if googlebot_index != -1 and not allow_rule_present:
+        # Trova l'indice della fine della sezione User-Agent: Googlebot
+        end_googlebot_index = googlebot_index + 1
+        while (end_googlebot_index < len(lines) and 
+            not lines[end_googlebot_index]
+             .lower().replace(' ', '')
+             .startswith(normalized_useragent_string)):
+            end_googlebot_index += 1
+
+        # Aggiungi Allow: /*?expand* alla fine della sezione User-Agent: Googlebot
+        if lines[end_googlebot_index -1] == '':
+            end_googlebot_index -= 1
+        lines.insert(end_googlebot_index, googlebot_allow_rule)
+
+        lines = '\n'.join(lines)
+        api.portal.set_registry_record('plone.robots_txt', lines)
+        logger.info("Upgrade robots.txt with rule for googlebot")
+    else:
+        logger.info("Rule for Googlebot already present in robots.txt, no action needed")
+

src/redturtle/volto/upgrades.zcml

@@ -180,5 +180,13 @@
       handler=".upgrades.to_4200"
       />
 
+  <genericsetup:upgradeStep
+      title="Add controlpanel"
+      description=""
+      profile="redturtle.volto:default"
+      source="4200"
+      destination="4300"
+      handler=".upgrades.upgrade_robots_txt"
+      />
 
 </configure>