more books

.gitignore

@@ -1,3 +1,5 @@
 build
 node_modules
-.envrc
+.envrc
+googleServiceAccount.json
+stats.json

books/Its Not Luck.yml

@@ -0,0 +1,5 @@
+---
+book:
+  isbn: 9780566076275
+  tags:
+    - management

classes/GitFile.js

@@ -0,0 +1,19 @@
+const Git = require("nodegit")
+
+class GitFile {
+    constructor(location) {
+        this.location = location
+    }
+    revs(count, order = Git.Revwalk.SORT.REVERSE) {
+        return Git.Repository
+            .open(".")
+            .then(repo => repo.getMasterCommit().then(master => ([repo, master])))
+            .then(([repo, master]) => {
+                let walker = repo.createRevWalk()
+                walker.push(master.sha())
+                walker.sorting(order)
+                return walker.fileHistoryWalk(this.location, count)})
+    }
+}
+
+module.exports = GitFile

generateFeed.js

@@ -0,0 +1,45 @@
+import { Feed } from "feed"
+import GitFile from './classes/GitFile'
+
+const max = (among, attr) => new Date(Math.max(...among.map(item => attr(item))))
+
+const emptyFeed = new Feed({
+    title: 'The best books for developers – mustread.tech',
+    description: "Open-source and crowd-sourced book listing",
+    link: "http://mustread.tech",
+    id: "http://static.mustread.tech/rss.xml",
+    author: {
+        name: "Eduards Sizovs"
+    },
+})
+
+module.exports = books => 
+    Promise
+        .all(books
+            .map(book => 
+                    new GitFile(book.location)
+                    .revs(1)
+                    .then(([firstRev]) => {
+                        if (!firstRev) {
+                            console.warn('😵 Cannot get git history of ' + book.location)
+                            return book
+                        } else {
+                            return { ...book, added: firstRev.commit.date() }
+                        }
+                    })
+                )
+            )
+        .then(stats => stats
+            .filter(stat => stat)
+            .reduce((feed, book) => {
+                feed.addItem({
+                    id: book.objectID,
+                    date: book.added,
+                    link: "http://mustread.tech/books/isbn/" + book.objectID, 
+                    title: "Must-read book: " + book.title,
+                    description: book.description.slice(0, 256) + "..."
+                })
+                feed.options.updated = max(feed.items, item => item.date)                
+                return feed
+            }, emptyFeed))
+        .then(feed => feed.atom1())

generateStats.js

@@ -0,0 +1,28 @@
+const dayjs = require('dayjs')
+const GitFile = require('./classes/GitFile')
+
+const commitMonth = rev => dayjs(rev.commit.date()).format('MMMM')
+
+module.exports = books => 
+    Promise
+        .all(books
+            .map(book => 
+                    new GitFile(book.location)
+                    .revs(1)
+                    .then(([firstRev]) => {
+                        let isbn = book.objectID
+                        if (!firstRev) {
+                            console.warn('😵 Cannot get git history of ' + book.location)
+                            return undefined
+                        } else {
+                            return { isbn: isbn.toString(), month: commitMonth(firstRev) }
+                        }
+                    })
+                )
+            )
+        .then(stats => stats
+            .filter(stat => stat)
+            .reduce((acc, curr) => 
+                Object.assign(acc, {[curr.month]: [curr.isbn].concat(acc[curr.month] || [])}), {}
+            ))
+        .then(JSON.stringify)

indexAll.js

@@ -18,16 +18,17 @@
     const algolia = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_API_KEY)
     const index = algolia.initIndex(process.env.ALGOLIA_INDEX_NAME)
 
-    const fileNames = await globAsync('./books/*.yml')
+    const generateStats = require("./generateStats")
+    const generateFeed = require("./generateFeed")
+    const store = require('./store')
 
-    const contributions = fileNames
-                            .map(readFileUtf8)
-                            .map(yaml.load)
-
-    contributions.forEach(async contribution => {
-    // contributions.filter(c => c.book.isbn == '9781934356852').forEach(async contribution => {
+    const contributedFiles = (await globAsync('books/*.yml'))
+                                    .map(contributedFile => [contributedFile, readFileUtf8(contributedFile)])
+                                    .map(([contributedFile, content]) => ({...yaml.load(content), location: contributedFile}) )
+              
+    const outIfNeeded = c => true
+    const booksInFuture = contributedFiles.filter(outIfNeeded).map(async contribution => {
         const { isbn, description, tags } = contribution.book
-
         try {
 
             const goodreads = await require('./fetch/goodreads')(isbn)
@@ -36,7 +37,9 @@
 
             const book = {
                 objectID: isbn,
+                isbn: isbn,
                 about: tags,
+                location: contribution.location,
                 title: goodreads.title,
                 description: description || goodreads.description || gbooks.description,
                 ratingCount: goodreads.ratingCount,
@@ -49,10 +52,21 @@
                 year: goodreads.year || gbooks.year
             }
 
-            await index.saveObject(book)
+            return book
         } catch (e) {
-            console.error(`😵 Sorry, things went wrong at ${isbn}.`, e)
+            console.error(`😵 Sorry, things went fetching ${isbn}.`, e)
+            return undefined
         }
     })
+
+
+
+    const books = (await Promise.all(booksInFuture)).filter(book => book)
+    await index.saveObjects(books)
+
+    const stats = await generateStats(books)
+    const feed = await generateFeed(books)
+
+    await store(stats, feed)
 })()
 

package.json

@@ -5,15 +5,18 @@
   "author": "Eduards Sizovs <[email protected]>",
   "private": true,
   "scripts": {
-    "start": "babel . --out-dir ./build --ignore 'node_modules' && node ./build/indexAll.js"
+    "start": "babel . --out-dir ./build --ignore 'node_modules','build' && node ./build/indexAll.js"
   },
   "dependencies": {
+    "@google-cloud/storage": "^2.3.4",
     "algoliasearch": "^3.32.0",
     "axios": "^0.18.0",
     "babel-register": "^6.26.0",
     "cheerio": "^1.0.0-rc.2",
     "dayjs": "^1.7.8",
+    "feed": "^2.0.2",
     "js-yaml": "^3.12.0",
+    "nodegit": "^0.23.0",
     "xml-js": "^1.6.8"
   },
   "devDependencies": {

store.js

@@ -0,0 +1,35 @@
+const { Storage } = require('@google-cloud/storage');
+
+const BUCKET_NAME = 'static.mustread.tech'
+
+const storage = new Storage({ projectId: 'mustread-tech' }) 
+const bucket = storage.bucket(BUCKET_NAME)
+
+const saveFile = async (name, content, contentType) => {
+  const file = bucket.file(name)
+  await file.save(content)
+  await file.setMetadata({ cacheControl: 'no-cache', contentType: contentType, contentDisposition: `inline; filename="${name}"` })
+  await file.makePublic() 
+}
+
+module.exports = async (stats, rss) => {
+
+  const bucketExists = await bucket.exists()
+  if (!bucketExists) {
+    await bucket.create()
+  }
+
+  await bucket.setMetadata({
+    cors: [
+      {
+        origin: ['*'],
+        method: ['*'],
+        responseHeader: ["Content-Type"],
+        maxAgeSeconds: 3600
+      }
+    ]
+  })
+
+  await saveFile('stats.json', stats, 'application/json')
+  await saveFile('rss.xml', rss, 'application/xml')
+}