Merge pull request #207 from nih-sparc/sitemap-robots-in-work

Updated sitemap and robots modules

nuxt.config.ts

@@ -203,20 +203,30 @@ export default defineNuxtConfig({
   */
   css: ['sparc-design-system-components-2/dist/style.css', '@/assets/_base.scss'],
   sitemap: {
+    cacheMaxAgeSeconds: 86400,
     sources: [
       '/api/__sitemap__/urls'
     ],
+    exclude: [
+      '/datasets/plotviewer',
+      '/datasets/simulationviewer',
+      '/datasets/timeseriesviewer',
+      '/datasets/videoviewer',
+      '/datasets/biolucidaviewer',
+      '/datasets/flatmapviewer',
+      '/datasets/imageviewer',
+      '/datasets/scaffoldviewer',
+    ],
+
     xslColumns: [
       { label: 'URL', width: '100%' }
     ],
   },
   robots: {
-    sitemap: 'https://sparc.science/sitemap.xml',
     // provide simple disallow rules for all robots `user-agent: *`
     // disallowing certain pages that are either redirects, authticated routes, or causing bots to recursively crawl
     disallow: [
-      '/datasets',
-      '/data',
+      '/datasets/*?*',
       '/welcome', 
       '/user', 
       '/contact-us', 
@@ -227,6 +237,6 @@ export default defineNuxtConfig({
       '/news-and-events/community-spotlight/submit'
     ],
     blockNonSeoBots: true,
-    crawlDelay: 60
+    crawlDelay: 3600
   }
 })

package.json

@@ -8,7 +8,6 @@
     "preview": "nuxt preview",
     "start": "node .output/server/index.mjs",
     "postinstall": "nuxt prepare",
-    "sitemap": "node generate_sitemap.js",
     "test-cypress": "cypress run",
     "test-cypress-record": "cypress run --record"
   },
@@ -20,8 +19,7 @@
     "nuxt": "^3.8.2",
     "nuxt-simple-robots": "^4.0.0-rc.19",
     "nuxt-svgo": "^3.5.6",
-    "sass": "^1.66.1",
-    "sitemap": "^7.1.1"
+    "sass": "^1.66.1"
   },
   "dependencies": {
     "@abi-software/mapintegratedvuer": "1.5.0",

server/api/__sitemap__/urls.js

@@ -4,7 +4,7 @@ import { defineSitemapEventHandler } from '#imports'
 
 export default defineSitemapEventHandler(async () => {
   try {
-    const PORTAL_BASE_URL = process.env.ROOL_URL || 'https://staging.sparc.science'
+    const PORTAL_BASE_URL = process.env.ROOT_URL
 
     // Algolia creds
     const ALGOLIA_APP_ID = process.env.ALGOLIA_APP_ID
@@ -159,21 +159,10 @@ export default defineSitemapEventHandler(async () => {
       }
     })
 
-    // Fetching ABOUT DETAILS IDs
-
-    const aboutResp = await contentfulClient.getEntries({
-      content_type: 'aboutPageSecondLevel',
-      limit: 1000
-    })
-
-    aboutResp.items.forEach(db => {
-      if (db.sys.id) {
-        urls.push(PORTAL_BASE_URL + '/about/' + db.sys.id)
-      }
-    })
-
     return urls.map((url) => {
-      return { loc: url }
+      // strip any parameters set on the url
+      const parsedUrl = new URL(url)
+      return { loc: parsedUrl.origin + parsedUrl.pathname }
     })
   } catch (err) {
     console.error('Error fetching dynamic routes:', err)

yarn.lock

@@ -6731,11 +6731,6 @@
   dependencies:
     undici-types "~5.26.4"
 
-"@types/node@^17.0.5":
-  version "17.0.45"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
-  integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
-
 "@types/node@^18.0.0":
   version "18.19.33"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.33.tgz#98cd286a1b8a5e11aa06623210240bcc28e95c48"
@@ -6783,13 +6778,6 @@
   resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.2.tgz#97d26e00cd4a0423b4af620abecf3e6f442b7975"
   integrity sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==
 
-"@types/sax@^1.2.1":
-  version "1.2.7"
-  resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d"
-  integrity sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==
-  dependencies:
-    "@types/node" "*"
-
 "@types/semver@^7.3.4":
   version "7.5.8"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e"
@@ -7809,11 +7797,6 @@ are-we-there-yet@^2.0.0:
     delegates "^1.0.0"
     readable-stream "^3.6.0"
 
-arg@^5.0.0:
-  version "5.0.2"
-  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
-  integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
-
 argparse@^1.0.10:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -18686,16 +18669,6 @@ [email protected], site-config-stack@^2.2.12:
   dependencies:
     ufo "^1.4.0"
 
-sitemap@^7.1.1:
-  version "7.1.1"
-  resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-7.1.1.tgz#eeed9ad6d95499161a3eadc60f8c6dce4bea2bef"
-  integrity sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==
-  dependencies:
-    "@types/node" "^17.0.5"
-    "@types/sax" "^1.2.1"
-    arg "^5.0.0"
-    sax "^1.2.4"
-
 slash@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"