ZFIN-9404: Add directives to tell crawlers not to crawl URLs with tem…

…porary IDs for phenotype statements. (ZFIN#1190)
rtaylorzfin · Oct 25, 2024 · 4427827 · 4427827
1 parent abed714
commit 4427827
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 3 deletions.
diff --git a/home/WEB-INF/jsp/ontology/phenotype-summary.jsp b/home/WEB-INF/jsp/ontology/phenotype-summary.jsp
@@ -45,7 +45,7 @@
                                     <a href="/action/phenotype/statement/${phenotype.id}">
                                             ${phenotype.displayName}
                                     </a>
-                                    <a href="/action/phenotype/statement-popup/${phenotype.id}" class="popup-link data-popup-link"></a>
+                                    <a href="/action/phenotype/statement-popup/${phenotype.id}" class="popup-link data-popup-link" rel="nofollow"></a>
                                     <div></div>
                                 </c:forEach>
                             </td>

diff --git a/home/javascript/react/components/PhenotypeStatement.js b/home/javascript/react/components/PhenotypeStatement.js
@@ -10,10 +10,12 @@ const PhenotypeStatement = ({statement}) => {
                 key={statement.id}
                 href={'/action/phenotype/statement/' + statement.id}
                 dangerouslySetInnerHTML={{__html: statement.shortName}}
+                rel='nofollow'
             />
             <a
                 className='popup-link data-popup-link'
                 href={`/action/phenotype/statement-popup/${statement.id}`}
+                rel='nofollow'
             />
         </div>
     )

diff --git a/home/javascript/react/components/entity/PhenotypeStatementLink.js b/home/javascript/react/components/entity/PhenotypeStatementLink.js
@@ -5,8 +5,8 @@ import {entityIDType} from '../../utils/types';
 const PhenotypeStatementLink = ({entity}) => {
     return (
         <>
-            <a href={`/action/phenotype/statement/${entity.id}`} dangerouslySetInnerHTML={{__html: entity.shortName}} />
-            <a href={`/action/phenotype/statement-popup/${entity.id}`} className='popup-link data-popup-link'/>
+            <a href={`/action/phenotype/statement/${entity.id}`} dangerouslySetInnerHTML={{__html: entity.shortName}} rel='nofollow' />
+            <a href={`/action/phenotype/statement-popup/${entity.id}`} className='popup-link data-popup-link' rel='nofollow'/>
         </>
     );
 };

diff --git a/home/robots.txt b/home/robots.txt
@@ -39,6 +39,10 @@ Disallow: /action/quicksearch
 #bots should not do updates or submit feedback
 Disallow: /cgi-bin/webdriver?MIval=aa-your_input_welcome.apg
 
+#bots don't need to crawl temporary phenotype statement pages (IDs are temporary):
+Disallow: /action/phenotype/statement/
+Disallow: /action/phenotype/statement-popup/
+
 Disallow: /downloads/archive
 Disallow: /downloads/archive/*