diff --git a/.gitignore b/.gitignore index baf9d297..90094423 100644 --- a/.gitignore +++ b/.gitignore @@ -134,4 +134,7 @@ version # Ignore IDE files .idea -.vscode \ No newline at end of file +.vscode + +# Sitemaps +backend/public/*-sitemap.xml \ No newline at end of file diff --git a/.woodpecker/.docker.yaml b/.woodpecker/.docker.yaml index ac903fc6..027f34db 100644 --- a/.woodpecker/.docker.yaml +++ b/.woodpecker/.docker.yaml @@ -55,6 +55,3 @@ steps: tag: pull_${CI_COMMIT_PULL_REQUEST} when: event: pull_request - -depends_on: - - test diff --git a/.woodpecker/.test.yaml b/.woodpecker/.test.yaml deleted file mode 100644 index 5721766d..00000000 --- a/.woodpecker/.test.yaml +++ /dev/null @@ -1,73 +0,0 @@ -when: - - event: pull_request - - event: push - branch: - - ${CI_REPO_DEFAULT_BRANCH} - -variables: - - &node_image "node:18" - -services: - mongodb: - image: mongo - environment: - MONGO_INITDB_ROOT_USERNAME: admin - MONGO_INITDB_ROOT_PASSWORD: password - ports: ["27017"] - -steps: - Generate version file: - image: alpine/git - commands: - - git describe --always --tags > version - - cp version backend/version - - Install Backend Dependencies: - image: *node_image - commands: - - cd backend - - npm install - - cd .. - - Install Frontend Dependencies: - image: *node_image - commands: - - cd frontend - - npm install - - cd .. - - Test Backend: - image: *node_image - environment: - SESSION_SECRET: 64ee54e91d5ea65c2fff46f7bf0d6b2c - DB_NAME: camphouse - DB_HOST: mongodb - DB_PORT: "27017" - DB_USER: admin - DB_PASS: password - JWT_SECRET: bff572ca273261b7bd0cb4a2f201913e - commands: - - cd backend - - npm test - - cd .. - - Test Frontend: - image: *node_image - commands: - - cd frontend - - npm run test:unit - - cd .. - - Lint Backend: - image: *node_image - commands: - - cd backend - - npm run lint - - cd .. - - Lint Frontend: - image: *node_image - commands: - - cd frontend - - npm run lint - - cd .. diff --git a/backend/app.js b/backend/app.js index cc7fb56f..0679858d 100644 --- a/backend/app.js +++ b/backend/app.js @@ -4,13 +4,20 @@ const express = require("express"); const rateLimit = require("express-rate-limit"); const mongoStore = require("rate-limit-mongo"); const helmet = require("helmet"); -const fs = require("fs"); +const cors = require("cors"); +const { fs } = require("fs"); const chalk = require("chalk"); const swaggerJsdoc = require("swagger-jsdoc"); const swaggerUi = require("swagger-ui-express"); const { getVersion } = require("./utils/general"); const path = require("path"); const fileUpload = require("express-fileupload"); +const { + updatePostsSitemap, + updateUsersSitemap, + USERS_SITEMAP_FILE, + POSTS_SITEMAP_FILE, +} = require("./controllers/sitemap"); // Initialize Exceptionless let Exceptionless; @@ -29,6 +36,8 @@ require("dotenv").config(); const app = express(); const port = process.env.APP_PORT || 3000; +app.use(cors()); + // Begin the server (async () => { // Initialize Exceptionless @@ -189,6 +198,8 @@ const port = process.env.APP_PORT || 3000; app.use(express.json()); // Used to parse the form data that is sent to the server app.use(express.urlencoded({ extended: true })); + // Public directory + app.use(express.static(path.join(__dirname, "public"))); // Load the controllers const PostController = require("./controllers/PostController"); @@ -220,6 +231,35 @@ const port = process.env.APP_PORT || 3000; app.use("/api/v1/blocked-email-domains", BlockedEmailDomainController); app.use("/api/v1/audit-logs", AuditLogController); + // Custom routes to serve the sitemap files + app.get("/api/users-sitemap.xml", async (req, res) => { + res.sendFile(USERS_SITEMAP_FILE, (err) => { + if (err) { + console.error("Error sending users sitemap file:", err); + res.status(500).json({ + status: "error", + code: 500, + message: "Internal server error", + data: null, + }); + } + }); + }); + + app.get("/api/posts-sitemap.xml", async (req, res) => { + res.sendFile(POSTS_SITEMAP_FILE, (err) => { + if (err) { + console.error("Error sending users sitemap file:", err); + res.status(500).json({ + status: "error", + code: 500, + message: "Internal server error", + data: null, + }); + } + }); + }); + // Swagger documentation const options = require("./configs/swagger"); const specs = swaggerJsdoc(options); @@ -278,5 +318,16 @@ const port = process.env.APP_PORT || 3000; console.log( chalk.yellow(`📚 API docs at http://localhost:${port}/api/docs`) ); + + setTimeout(() => { + updatePostsSitemap(); + updateUsersSitemap(); + }, 5000); + + // Recheck every 10 minutes + setInterval(() => { + updatePostsSitemap(); + updateUsersSitemap(); + }, 600000); }); })(); diff --git a/backend/controllers/sitemap.js b/backend/controllers/sitemap.js new file mode 100644 index 00000000..b497ad5b --- /dev/null +++ b/backend/controllers/sitemap.js @@ -0,0 +1,87 @@ +const { createWriteStream } = require("fs"); +const axios = require("axios"); +const { SitemapStream, streamToPromise } = require("sitemap"); +const path = require("path"); + +const USERS_SITEMAP_FILE = path.join(__dirname, "../public", "users-sitemap.xml"); +const POSTS_SITEMAP_FILE = path.join(__dirname, "../public", "posts-sitemap.xml"); + +const updateUsersSitemap = async () => { + try { + const response = await axios.get( + "https://camphouse.vmgware.dev/api/v1/users?page=1&limit=1000" + ); + const userData = response.data; + const users = userData.data.users; + + const sitemapStream = new SitemapStream({ + hostname: "https://camphouse.vmgware.dev/", + lastmodDateOnly: false, + }); + + for (const user of users) { + const userHandle = user.handle; + const userLink = `https://camphouse.vmgware.dev/@${userHandle}`; + + sitemapStream.write({ + url: userLink, + changefreq: "weekly", + lastmod: user.updatedAt, + }); + } + + sitemapStream.end(); + const sitemapXML = await streamToPromise(sitemapStream); + const sitemapFile = createWriteStream(USERS_SITEMAP_FILE); + sitemapFile.write(sitemapXML.toString()); + sitemapFile.end(); + + console.log("Users sitemap updated successfully!"); + } catch (error) { + console.error("Failed to update users sitemap:", error); + } +}; + +// Function to update the posts sitemap +const updatePostsSitemap = async () => { + try { + const response = await axios.get( + "https://camphouse.vmgware.dev/api/v1/posts?page=1&limit=1000" + ); + const postData = response.data; + const posts = postData.data.posts; + + const sitemapStream = new SitemapStream({ + hostname: "https://camphouse.vmgware.dev/", + lastmodDateOnly: false, + }); + + for (const post of posts) { + const postId = post._id; + const postLink = `https://camphouse.vmgware.dev/post/${postId}`; + + sitemapStream.write({ + url: postLink, + changefreq: "daily", + lastmod: post.updatedAt, + }); + } + + sitemapStream.end(); + const sitemapXML = await streamToPromise(sitemapStream); + const sitemapFile = createWriteStream(POSTS_SITEMAP_FILE); + sitemapFile.write(sitemapXML.toString()); + sitemapFile.end(); + + console.log("Posts sitemap updated successfully!"); + } catch (error) { + console.error("Failed to update posts sitemap:", error); + } +}; + +module.exports = { + updateUsersSitemap, + updatePostsSitemap, + USERS_SITEMAP_FILE, + POSTS_SITEMAP_FILE, +}; diff --git a/backend/package-lock.json b/backend/package-lock.json index 6dd0fc86..0864e10d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,10 +10,12 @@ "license": "ISC", "dependencies": { "@exceptionless/node": "^3.0.5", - "@sentry/node": "^7.91.0", + "@sentry/node": "^7.93.0", + "axios": "^1.7.2", "bcrypt": "^5.1.0", "chai-http": "^4.4.0", "chalk": "^4.1.0", + "cors": "^2.8.5", "dotenv": "^16.1.4", "express": "^4.18.2", "express-fileupload": "^1.4.3", @@ -27,6 +29,7 @@ "qrcode": "^1.5.3", "rate-limit-mongo": "^2.3.2", "sinon-chai": "^3.7.0", + "sitemap": "^7.1.1", "speakeasy": "^2.0.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0" @@ -1059,39 +1062,39 @@ } }, "node_modules/@sentry-internal/tracing": { - "version": "7.91.0", - "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.91.0.tgz", - "integrity": "sha512-JH5y6gs6BS0its7WF2DhySu7nkhPDfZcdpAXldxzIlJpqFkuwQKLU5nkYJpiIyZz1NHYYtW5aum2bV2oCOdDRA==", + "version": "7.93.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.93.0.tgz", + "integrity": "sha512-DjuhmQNywPp+8fxC9dvhGrqgsUb6wI/HQp25lS2Re7VxL1swCasvpkg8EOYP4iBniVQ86QK0uITkOIRc5tdY1w==", "dependencies": { - "@sentry/core": "7.91.0", - "@sentry/types": "7.91.0", - "@sentry/utils": "7.91.0" + "@sentry/core": "7.93.0", + "@sentry/types": "7.93.0", + "@sentry/utils": "7.93.0" }, "engines": { "node": ">=8" } }, "node_modules/@sentry/core": { - "version": "7.91.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.91.0.tgz", - "integrity": "sha512-tu+gYq4JrTdrR+YSh5IVHF0fJi/Pi9y0HZ5H9HnYy+UMcXIotxf6hIEaC6ZKGeLWkGXffz2gKpQLe/g6vy/lPA==", + "version": "7.93.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.93.0.tgz", + "integrity": "sha512-vZQSUiDn73n+yu2fEcH+Wpm4GbRmtxmnXnYCPgM6IjnXqkVm3awWAkzrheADblx3kmxrRiOlTXYHw9NTWs56fg==", "dependencies": { - "@sentry/types": "7.91.0", - "@sentry/utils": "7.91.0" + "@sentry/types": "7.93.0", + "@sentry/utils": "7.93.0" }, "engines": { "node": ">=8" } }, "node_modules/@sentry/node": { - "version": "7.91.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.91.0.tgz", - "integrity": "sha512-hTIfSQxD7L+AKIqyjoq8CWBRkEQrrMZmA3GSZgPI5JFWBHgO0HBo5TH/8TU81oEJh6kqqHAl2ObMhmcnaFqlzg==", - "dependencies": { - "@sentry-internal/tracing": "7.91.0", - "@sentry/core": "7.91.0", - "@sentry/types": "7.91.0", - "@sentry/utils": "7.91.0", + "version": "7.93.0", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.93.0.tgz", + "integrity": "sha512-nUXPCZQm5Y9Ipv7iWXLNp5dbuyi1VvbJ3RtlwD7utgsNkRYB4ixtKE9w2QU8DZZAjaEF6w2X94OkYH6C932FWw==", + "dependencies": { + "@sentry-internal/tracing": "7.93.0", + "@sentry/core": "7.93.0", + "@sentry/types": "7.93.0", + "@sentry/utils": "7.93.0", "https-proxy-agent": "^5.0.0" }, "engines": { @@ -1099,19 +1102,19 @@ } }, "node_modules/@sentry/types": { - "version": "7.91.0", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.91.0.tgz", - "integrity": "sha512-bcQnb7J3P3equbCUc+sPuHog2Y47yGD2sCkzmnZBjvBT0Z1B4f36fI/5WjyZhTjLSiOdg3F2otwvikbMjmBDew==", + "version": "7.93.0", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.93.0.tgz", + "integrity": "sha512-UnzUccNakhFRA/esWBWP+0v7cjNg+RilFBQC03Mv9OEMaZaS29zSbcOGtRzuFOXXLBdbr44BWADqpz3VW0XaNw==", "engines": { "node": ">=8" } }, "node_modules/@sentry/utils": { - "version": "7.91.0", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.91.0.tgz", - "integrity": "sha512-fvxjrEbk6T6Otu++Ax9ntlQ0sGRiwSC179w68aC3u26Wr30FAIRKqHTCCdc2jyWk7Gd9uWRT/cq+g8NG/8BfSg==", + "version": "7.93.0", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.93.0.tgz", + "integrity": "sha512-Iovj7tUnbgSkh/WrAaMrd5UuYjW7AzyzZlFDIUrwidsyIdUficjCG2OIxYzh76H6nYIx9SxewW0R54Q6XoB4uA==", "dependencies": { - "@sentry/types": "7.91.0" + "@sentry/types": "7.93.0" }, "engines": { "node": ">=8" @@ -1734,6 +1737,14 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/superagent": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.13.tgz", @@ -1916,6 +1927,11 @@ "node": ">=10" } }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1965,6 +1981,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2414,6 +2440,18 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -3059,6 +3097,25 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -3438,9 +3495,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ip": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", - "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", + "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==", "optional": true, "peer": true }, @@ -4578,6 +4635,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -5208,6 +5270,29 @@ "node": ">=4" } }, + "node_modules/sitemap": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-7.1.1.tgz", + "integrity": "sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==", + "dependencies": { + "@types/node": "^17.0.5", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.2.4" + }, + "bin": { + "sitemap": "dist/cli.js" + }, + "engines": { + "node": ">=12.0.0", + "npm": ">=5.6.0" + } + }, + "node_modules/sitemap/node_modules/@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==" + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", diff --git a/backend/package.json b/backend/package.json index 99245afb..415bccb6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,10 +16,12 @@ "license": "ISC", "dependencies": { "@exceptionless/node": "^3.0.5", - "@sentry/node": "^7.91.0", + "@sentry/node": "^7.93.0", + "axios": "^1.7.2", "bcrypt": "^5.1.0", "chai-http": "^4.4.0", "chalk": "^4.1.0", + "cors": "^2.8.5", "dotenv": "^16.1.4", "express": "^4.18.2", "express-fileupload": "^1.4.3", @@ -33,6 +35,7 @@ "qrcode": "^1.5.3", "rate-limit-mongo": "^2.3.2", "sinon-chai": "^3.7.0", + "sitemap": "^7.1.1", "speakeasy": "^2.0.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0" @@ -45,4 +48,4 @@ "mocha": "^10.2.0", "sinon": "^7.5.0" } -} \ No newline at end of file +} diff --git a/frontend/public/index.html b/frontend/public/index.html index 01fc5c9c..d5908b84 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -10,6 +10,33 @@ + + + + + + + + + + + + + +