itsalive.coffee

# Turn a mongo db into a live-db
# you should probably redis-cli flushdb; before running this script
async = require 'async'
argv = require('optimist').argv
backend = require './backend'
livedb = require 'livedb'
LiveDbMongo = require 'livedb-mongo'

consoleError = (err) ->
  return console.error err if err

# Taken from livedb-mongo/mongo.js
shallowClone = (object) ->
  out = {}
  for key of object
    out[key] = object[key]
  return out

castToDoc = (docName, data) ->
  doc = if (typeof data.data == 'object' && data.data != null && !Array.isArray(data.data))
    shallowClone(data.data)
  else
    _data: if data.data is undefined then null else data.data
  doc._type = data.type || null
  doc._v = data.v
  doc._id = docName
  return doc

castToSnapshot = (doc) ->
  return if !doc
  type = doc._type
  v = doc._v
  docName = doc._id
  data = doc._data
  if data is undefined
    doc = shallowClone(doc)
    delete doc._type
    delete doc._v
    delete doc._id
    return {
      data: doc
      type: type
      v: v
      docName: docName
    }
  return {
    data: data
    type: type
    v: v
    docName: docName
  }


exports = module.exports
exports.itsalive = (options = {}, callback) ->
  #don't need to make these collections live
  blacklist = ['system.indexes', 'system.profile', 'system.users', 'system.js', 'configs', 'sessions']
  batch = options.b or argv.b

  # Setup the db
  mongo = backend.createMongo options
  db = new LiveDbMongo mongo
  redis = backend.createRedis options
  redisObserver = backend.createRedis options
  driver = livedb.redisDriver db, redis, redisObserver
  ldbc = livedb.client
    db: db
    driver: driver
    #redis: backend.createRedis options
    #redisObserver: backend.createRedis options

  ottypes = require('ottypes')
  jsonType = ottypes.json0.uri

  mongo.collections (err, cols) ->
    return callback err if err
    collections = []
    cols.forEach (c) ->
      console.log "collection", c.collectionName
      if not ~blacklist.indexOf(c.collectionName) and not /_ops$/.test c.collectionName
        collections.push c

    # Each must be used for large datasets, toArray won't fit in memory: http://mongodb.github.io/node-mongodb-native/api-generated/cursor.html#each
    async.each collections, (collection, cb) ->
      #go through all of the docs in the collection and create an op log for it
      cName = collection.collectionName
      mongo.collection(cName + "_ops").ensureIndex {name: 1, v: 1}, false, consoleError
      collection.count (err, count) ->
        return cb err if err
        console.log cName, "DOCS", count
        counter = 0

        cursor = collection.find()
        cursor.batchSize(+batch) if batch
        cursor.each (err, doc) ->
          if err
            console.log "uh oh", cName, err
            return cb err
          #we are done when we get a null doc
          if !doc
            cb(null, counter) if counter == count
            return

          doc._type ?= jsonType
          snapshot = castToSnapshot doc
          docName = snapshot.docName

          handleError = (err, opts, ecb) ->
            console.log "Error with #{cName} #{docName}"
            console.error err
            console.log "doc", JSON.stringify(doc)
            console.log "opts", JSON.stringify(opts)
            return ecb(err)

          #console.log "doc", counter, cName, docName
          db.getVersion cName, docName, (err, opVersion) ->
            return handleError err, {fn: "getVersion"}, cb if err

            if opVersion is 0
              console.log "no ops", cName, docName, counter
              # Create an operation that creates the document snapshot
              opData = create: {type: jsonType, data: snapshot.data}, v:0

              db.writeOp cName, docName, opData, (err) ->
                return handleError err, {fn:"writeOp", opData}, cb if err

                snapshot.v = 1
                db.writeSnapshot cName, docName, snapshot, (err) ->
                  return handleError err, {fn:"writeSnapshot", snapshot}, cb if err
                  counter++
                  cb null, counter if counter is count

            else if opVersion isnt doc._v
              # replay oplog, which is the source of truth
              console.log "diff  ops", counter

              db.writeSnapshot cName, docName, {v:0}, (err) ->
                return handleError err, {fn:"writeSnapshot"}, cb if err
                ldbc.fetch cName, docName, (err, snapshot) ->
                  return handleError err, {fn:"fetch", snapshot}, cb if err
                  db.writeSnapshot cName, docName, snapshot, (err) ->
                    return handleError err, {fn:"writeSnapshot", snapshot}, cb if err
                    counter++
                    cb null, counter if counter is count

            else
              console.log "ok", cName, docName, counter
              counter++
              cb null, counter if counter is count

    , (err) ->
      console.log("callback?", err)
      return callback err if err
      db.close()
      callback()


#called directly from command line (not required as a module)
if require.main == module

  exports.itsalive null, (err) ->
    if err
      console.log "ERROR! NOT FINISHED!"
      console.log err
    else
      console.log "ALL DONE"
    process.exit()