Merge pull request #953 from solid/feature/index-update

Update index.html based on whether meta-tag is set or not

README.md

@@ -228,14 +228,21 @@ Run with:
 docker run -p 8443:8443 --name solid node-solid-server
 ```
 
-Modify the config as follows:
+This will enable you to login to solid on https://localhost:8443 and then create a new account
+but not yet use that account. After a new account is made you will need to create an entry for 
+it in your local (/etc/)hosts file in line with the account and subdomain i.e. 
+
+127.0.0.1	newsoliduser.localhost
+
+Then you'll be able to use solid as intended.
+
+You can modify the config within the docker container as follows:
 
  - Copy the config to the current directory with: `docker cp solid:/usr/src/app/config.json .`
  - Edit the `config.json` file
  - Copy the file back with `docker cp config.json solid:/usr/src/app/`
  - Restart the server with `docker restart solid`
 
-
 ## Library Usage
 
 ### Install Dependencies

bin/lib/cli-utils.js

@@ -0,0 +1,70 @@
+const fs = require('fs-extra')
+const { cyan, bold } = require('colorette')
+const { URL } = require('url')
+const LDP = require('../../lib/ldp')
+const AccountManager = require('../../lib/models/account-manager')
+const SolidHost = require('../../lib/models/solid-host')
+
+module.exports.getAccountManager = getAccountManager
+module.exports.loadAccounts = loadAccounts
+module.exports.loadConfig = loadConfig
+module.exports.loadUsernames = loadUsernames
+
+/**
+ * Returns an instance of AccountManager
+ *
+ * @param {Object} config
+ * @param {Object} [options]
+ * @returns {AccountManager}
+ */
+function getAccountManager (config, options = {}) {
+  const ldp = options.ldp || new LDP(config)
+  const host = options.host || SolidHost.from({ port: config.port, serverUri: config.serverUri })
+  return AccountManager.from({
+    host,
+    store: ldp,
+    multiuser: config.multiuser
+  })
+}
+
+function loadConfig (program, options) {
+  let argv = {
+    ...options,
+    version: program.version()
+  }
+  let configFile = argv['configFile'] || './config.json'
+
+  try {
+    const file = fs.readFileSync(configFile)
+
+    // Use flags with priority over config file
+    const config = JSON.parse(file)
+    argv = { ...config, ...argv }
+  } catch (err) {
+    // No file exists, not a problem
+    console.log(cyan(bold('TIP')), 'create a config.json: `$ solid init`')
+  }
+
+  return argv
+}
+
+/**
+ *
+ * @param root
+ * @param [serverUri] If not set, hostname must be set
+ * @param [hostname] If not set, serverUri must be set
+ * @returns {*}
+ */
+function loadAccounts ({ root, serverUri, hostname }) {
+  const files = fs.readdirSync(root)
+  hostname = hostname || new URL(serverUri).hostname
+  const isUserDirectory = new RegExp(`.${hostname}$`)
+  return files
+    .filter(file => isUserDirectory.test(file))
+}
+
+function loadUsernames ({ root, serverUri }) {
+  const hostname = new URL(serverUri).hostname
+  return loadAccounts({ root, hostname })
+    .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1))
+}

bin/lib/cli.js

@@ -3,6 +3,7 @@ const loadInit = require('./init')
 const loadStart = require('./start')
 const loadInvalidUsernames = require('./invalidUsernames')
 const loadMigrateLegacyResources = require('./migrateLegacyResources')
+const loadUpdateIndex = require('./updateIndex')
 const { spawnSync } = require('child_process')
 const path = require('path')
 
@@ -13,6 +14,7 @@ module.exports = function startCli (server) {
   loadStart(program, server)
   loadInvalidUsernames(program)
   loadMigrateLegacyResources(program)
+  loadUpdateIndex(program)
 
   program.parse(process.argv)
   if (program.args.length === 0) program.help()

bin/lib/invalidUsernames.js

@@ -1,17 +1,14 @@
 const fs = require('fs-extra')
 const Handlebars = require('handlebars')
 const path = require('path')
-const { URL } = require('url')
 
-const { loadConfig } = require('./common')
+const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils')
 const { isValidUsername } = require('../../lib/common/user-utils')
 const blacklistService = require('../../lib/services/blacklist-service')
 const { initConfigDir, initTemplateDirs } = require('../../lib/server-config')
 const { fromServerConfig } = require('../../lib/models/oidc-manager')
 
-const AccountManager = require('../../lib/models/account-manager')
 const EmailService = require('../../lib/services/email-service')
-const LDP = require('../../lib/ldp')
 const SolidHost = require('../../lib/models/solid-host')
 
 module.exports = function (program) {
@@ -28,7 +25,7 @@ module.exports = function (program) {
 
       const invalidUsernames = getInvalidUsernames(config)
       const host = SolidHost.from({ port: config.port, serverUri: config.serverUri })
-      const accountManager = getAccountManager(config, host)
+      const accountManager = getAccountManager(config, { host })
 
       if (options.notify) {
         return notifyUsers(invalidUsernames, accountManager, config)
@@ -96,23 +93,9 @@ async function deleteUsers (usernames, accountManager, config, host) {
   console.info(`Deleted ${deletingUsers.length} users succeeded`)
 }
 
-function getAccountManager (config, host) {
-  const ldp = new LDP(config)
-  return AccountManager.from({
-    host,
-    store: ldp,
-    multiuser: config.multiuser
-  })
-}
-
 function getInvalidUsernames (config) {
-  const files = fs.readdirSync(config.root)
-  const hostname = new URL(config.serverUri).hostname
-  const isUserDirectory = new RegExp(`.${hostname}$`)
-  return files
-    .filter(file => isUserDirectory.test(file))
-    .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1))
-    .filter(username => !isValidUsername(username) || !blacklistService.validate(username))
+  const usernames = loadUsernames(config)
+  return usernames.filter(username => !isValidUsername(username) || !blacklistService.validate(username))
 }
 
 function listUsernames (usernames) {

bin/lib/start.js

@@ -2,7 +2,7 @@
 
 const options = require('./options')
 const fs = require('fs')
-const { loadConfig } = require('./common')
+const { loadConfig } = require('./cli-utils')
 const { red, bold } = require('colorette')
 
 module.exports = function (program, server) {

bin/lib/updateIndex.js

@@ -0,0 +1,56 @@
+const fs = require('fs')
+const path = require('path')
+const cheerio = require('cheerio')
+const LDP = require('../../lib/ldp')
+const { URL } = require('url')
+const debug = require('../../lib/debug')
+const { readFile } = require('../../lib/common/fs-utils')
+
+const { compileTemplate, writeTemplate } = require('../../lib/common/template-utils')
+const { loadConfig, loadAccounts } = require('./cli-utils')
+const { getName, getWebId } = require('../../lib/common/user-utils')
+const { initConfigDir, initTemplateDirs } = require('../../lib/server-config')
+
+module.exports = function (program) {
+  program
+    .command('updateindex')
+    .description('Update index.html in root of all PODs that haven\'t been marked otherwise')
+    .action(async (options) => {
+      const config = loadConfig(program, options)
+      const configPath = initConfigDir(config)
+      const templates = initTemplateDirs(configPath)
+      const indexTemplatePath = path.join(templates.account, 'index.html')
+      const indexTemplate = await compileTemplate(indexTemplatePath)
+      const ldp = new LDP(config)
+      const accounts = loadAccounts(config)
+      const usersProcessed = accounts.map(async account => {
+        const accountDirectory = path.join(config.root, account)
+        const indexFilePath = path.join(accountDirectory, '/index.html')
+        if (!isUpdateAllowed(indexFilePath)) {
+          return
+        }
+        const accountUrl = getAccountUrl(account, config)
+        try {
+          const webId = await getWebId(accountDirectory, accountUrl, ldp.suffixMeta, (filePath) => readFile(filePath))
+          const name = await getName(webId, ldp.fetchGraph)
+          writeTemplate(indexFilePath, indexTemplate, { name, webId })
+        } catch (err) {
+          debug.errors(`Failed to create new index for ${account}: ${JSON.stringify(err, null, 2)}`)
+        }
+      })
+      await Promise.all(usersProcessed)
+      debug.accounts(`Processed ${usersProcessed.length} users`)
+    })
+}
+
+function getAccountUrl (name, config) {
+  const serverUrl = new URL(config.serverUri)
+  return `${serverUrl.protocol}//${name}.${serverUrl.host}/`
+}
+
+function isUpdateAllowed (indexFilePath) {
+  const indexSource = fs.readFileSync(indexFilePath, 'utf-8')
+  const $ = cheerio.load(indexSource)
+  const allowAutomaticUpdateValue = $('meta[name="solid-allow-automatic-updates"]').prop('content')
+  return !allowAutomaticUpdateValue || allowAutomaticUpdateValue === 'true'
+}

default-templates/new-account/index.html

@@ -3,6 +3,8 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+  <!-- Set content in the next line to false if you do not want your homepage to be automatically updated whenever a new version becomes available. -->
+  <meta name="solid-allow-automatic-updates" content="true">
   <title>{{#if name}}{{name}} &ndash; {{/if}}Solid Home</title>
   <link rel="stylesheet" href="/common/css/bootstrap.min.css">
   <link rel="stylesheet" href="/common/css/solid.css">

lib/common/fs-utils.js

@@ -1,5 +1,7 @@
 module.exports.copyTemplateDir = copyTemplateDir
 module.exports.processFile = processFile
+module.exports.readFile = readFile
+module.exports.writeFile = writeFile
 
 const fs = require('fs-extra')
 
@@ -31,3 +33,11 @@ async function processFile (filePath, manipulateSourceFn) {
     })
   })
 }
+
+function readFile (filePath, options = 'utf-8') {
+  return fs.readFileSync(filePath, options)
+}
+
+function writeFile (filePath, fileSource, options = 'utf-8') {
+  fs.writeFileSync(filePath, fileSource, options)
+}

lib/common/template-utils.js

@@ -1,8 +1,15 @@
+module.exports.compileTemplate = compileTemplate
 module.exports.processHandlebarFile = processHandlebarFile
+module.exports.writeTemplate = writeTemplate
 
 const Handlebars = require('handlebars')
 const debug = require('../debug').errors
-const { processFile } = require('./fs-utils')
+const { processFile, readFile, writeFile } = require('./fs-utils')
+
+async function compileTemplate (filePath) {
+  const indexTemplateSource = readFile(filePath)
+  return Handlebars.compile(indexTemplateSource)
+}
 
 /**
  * Reads a file, processes it (performing template substitution), and saves
@@ -36,3 +43,9 @@ function processHandlebarTemplate (source, substitutions) {
     return source
   }
 }
+
+function writeTemplate (filePath, template, substitutions) {
+  const source = template(substitutions)
+  writeFile(filePath, source)
+}
+

lib/common/user-utils.js

@@ -1,5 +1,28 @@
+const $rdf = require('rdflib')
+
+const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#')
+const VCARD = $rdf.Namespace('http://www.w3.org/2006/vcard/ns#')
+
+module.exports.getName = getName
+module.exports.getWebId = getWebId
 module.exports.isValidUsername = isValidUsername
 
+async function getName (webId, fetchGraph) {
+  const graph = await fetchGraph(webId)
+  const nameNode = graph.any($rdf.sym(webId), VCARD('fn'))
+  return nameNode.value
+}
+
+async function getWebId (accountDirectory, accountUrl, suffixMeta, fetchData) {
+  const metaFilePath = `${accountDirectory}/${suffixMeta}`
+  const metaFileUri = `${accountUrl}${suffixMeta}`
+  const metaData = await fetchData(metaFilePath)
+  const metaGraph = $rdf.graph()
+  $rdf.parse(metaData, metaGraph, metaFileUri, 'text/turtle')
+  const webIdNode = metaGraph.any(undefined, SOLID('account'), $rdf.sym(accountUrl))
+  return webIdNode.value
+}
+
 function isValidUsername (username) {
   return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(username)
 }

lib/handlers/index.js

@@ -3,6 +3,7 @@ module.exports = handler
 const path = require('path')
 const debug = require('debug')('solid:index')
 const Negotiator = require('negotiator')
+const url = require('url')
 
 async function handler (req, res, next) {
   const indexFile = 'index.html'
@@ -19,7 +20,7 @@ async function handler (req, res, next) {
     }
     // redirect to the right container if missing trailing /
     if (req.path.lastIndexOf('/') !== req.path.length - 1) {
-      return res.redirect(301, path.join(req.path, '/'))
+      return res.redirect(301, req.path + '/')
     }
 
     if (requestedType && requestedType.indexOf('text/html') !== 0) {
@@ -29,7 +30,7 @@ async function handler (req, res, next) {
 
     // Check if file exists in first place
     await ldp.exists(req.hostname, path.join(req.path, indexFile))
-    res.locals.path = path.join(req.path, indexFile)
+    res.locals.path = url.resolve(req.path, indexFile)
     debug('Found an index for current path')
   } catch (e) {
     // Ignore errors

lib/ldp.js

@@ -102,7 +102,7 @@ class LDP {
       const { path } = await this.resourceMapper.mapUrlToFile({ url })
       return await promisify(fs.readFile)(path, {'encoding': 'utf8'})
     } catch (err) {
-      throw error(404, "Can't read file")
+      throw error(err.status, err.message)
     }
   }
 
@@ -273,7 +273,7 @@ class LDP {
    *
    * @return {Promise<Graph>}
    */
-  async fetchGraph (uri, options) {
+  async fetchGraph (uri, options = {}) {
     const response = await fetch(uri)
     if (!response.ok) {
       const error = new Error(
@@ -284,7 +284,7 @@ class LDP {
     }
     const body = await response.text()
 
-    return promisify(parse)(body, uri, getContentType(response.headers))
+    return parse(body, uri, getContentType(response.headers))
   }
 
   /**