Feat: improve memory usage / allow for in-memory storage of all entries to be optional (WIP)

API.md

@@ -10,7 +10,7 @@ const Log = require('ipfs-log')
 
 ### Constructor
 
-#### new Log(ipfs, identity, [{ logId, access, entries, heads, clock, sortFn }])
+#### new Log(ipfs, identity, [{ logId, access, entries, heads, clock, sortFn, hashIndex }])
 
 Create a log. Each log gets a unique ID, which can be passed in the `options` as `logId`. Returns a `Log` instance.
 
@@ -32,15 +32,6 @@ console.log(log.id)
 
 Returns the ID of the log.
 
-#### values
-
-Returns an `Array` of [entries](https://github.com/orbitdb/ipfs-log/blob/master/src/entry.js) in the log. The values are in linearized order according to their [Lamport clocks](https://en.wikipedia.org/wiki/Lamport_timestamps).
-
-```javascript
-const values = log.values
-// TODO: output example
-```
-
 #### length
 
 Returns the number of entries in the log.
@@ -58,16 +49,25 @@ const heads = log.heads
 // TODO: output example
 ```
 
-#### tails
+### Methods
+
+#### values
 
-Return the tails of the log. Tails are the entries that reference other entries that are not in the log.
+Returns a *Promise* that resolves to an `Array` of [entries](https://github.com/orbitdb/ipfs-log/blob/master/src/entry.js) in the log. The values are in linearized order according to their [Lamport clocks](https://en.wikipedia.org/wiki/Lamport_timestamps).
 
 ```javascript
-const tails = log.tails
+const values = await log.values()
 // TODO: output example
 ```
 
-### Methods
+#### tails
+
+Returns a *Promise* that resolves to the tails of the log. Tails are the entries that reference other entries that are not in the log.
+
+```javascript
+const tails = await log.tails()
+// TODO: output example
+```
 
 #### append(data)
 
@@ -115,6 +115,18 @@ console.log(log.values)
 // ]
 ```
 
+#### get(hash)
+
+Returns a *Promise* that resolves to an entry if it exists, otherwise undefined.
+
+```javascript
+const entry = await get(hash)
+```
+
+#### has(hash)
+
+Returns true if the hash exists in the log, otherwise false.
+
 #### join(log, [length])
 
 Join the log with another log. Returns a Promise that resolves to a `Log` instance. The size of the joined log can be specified by giving `length` argument.
@@ -158,7 +170,7 @@ const buffer = log1.toBuffer()
 
 ### toString
 
-Returns the log values as a nicely formatted string.
+Returns a *Promise* that resolves to the log values as a nicely formatted string.
 
 ```javascript
 console.log(log.toString())

README.md

@@ -137,12 +137,14 @@ See [API Documentation](https://github.com/orbitdb/ipfs-log/tree/master/API.md)
     - [new Log(ipfs, identity, [{ logId, access, entries, heads, clock, sortFn }])](https://github.com/orbitdb/ipfs-log/tree/master/API.md##new-log-ipfs-id)
   - [Properties](https://github.com/orbitdb/ipfs-log/tree/master/API.md##properties)
     - [id](https://github.com/orbitdb/ipfs-log/tree/master/API.md##id)
-    - [values](https://github.com/orbitdb/ipfs-log/tree/master/API.md##values)
     - [length](https://github.com/orbitdb/ipfs-log/tree/master/API.md##length)
     - [clock](https://github.com/orbitdb/ipfs-log/tree/master/API.md##length)
     - [heads](https://github.com/orbitdb/ipfs-log/tree/master/API.md##heads)
-    - [tails](https://github.com/orbitdb/ipfs-log/tree/master/API.md##tails)
   - [Methods](https://github.com/orbitdb/ipfs-log/tree/master/API.md##methods)
+    - [values](https://github.com/orbitdb/ipfs-log/tree/master/API.md##values)
+    - [tails](https://github.com/orbitdb/ipfs-log/tree/master/API.md##tails)
+    - [get(hash)](https://github.com/orbitdb/ipfs-log/tree/master/API.md##get)
+    - [has(hash)](https://github.com/orbitdb/ipfs-log/tree/master/API.md##has)
     - [append(data)](https://github.com/orbitdb/ipfs-log/tree/master/API.md##appenddata)
     - [join(log)](https://github.com/orbitdb/ipfs-log/tree/master/API.md##joinlog)
     - [toMultihash()](https://github.com/orbitdb/ipfs-log/tree/master/API.md##tomultihash)

benchmarks/tail-hashes.js

@@ -15,7 +15,7 @@ const base = {
     return { log, ipfsd }
   },
   cycle: async function ({ log }) {
-    return log.tailHashes
+    return log.tailHashes()
   },
   teardown: async function ({ ipfsd }) {
     await stopIpfs(ipfsd)

benchmarks/tails.js

@@ -15,7 +15,7 @@ const base = {
     return { log, ipfsd }
   },
   cycle: async function ({ log }) {
-    return log.tails
+    return log.tails()
   },
   teardown: async function ({ ipfsd }) {
     await stopIpfs(ipfsd)

benchmarks/utils/create-log.js

@@ -17,7 +17,7 @@ const createLog = async (ipfs, logId) => {
   const access = new AccessController()
   const keystore = new Keystore(store)
   const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
-  const log = new Log(ipfs, identity, { logId: 'A', access })
+  const log = new Log(ipfs, identity, { logId: 'A', access, cacheSize: 5 })
   return { log, access, identity }
 }
 

benchmarks/values.js

@@ -15,7 +15,7 @@ const base = {
     return { log, ipfsd }
   },
   cycle: async function ({ log }) {
-    return log.values
+    return log.values()
   },
   teardown: async function ({ ipfsd }) {
     await stopIpfs(ipfsd)

lib/es5/entry-index.js

@@ -6,37 +6,45 @@ var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/cl
 
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 
+var LRU = require('lru-cache');
+
 var EntryIndex = /*#__PURE__*/function () {
   function EntryIndex() {
     var entries = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+    var cacheSize = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : Infinity;
     (0, _classCallCheck2["default"])(this, EntryIndex);
-    this._cache = entries;
+    this._cache = new LRU({
+      max: cacheSize
+    });
+    this.add(entries);
   }
 
   (0, _createClass2["default"])(EntryIndex, [{
     key: "set",
     value: function set(k, v) {
-      this._cache[k] = v;
+      this._cache.set(k, v);
     }
   }, {
     key: "get",
     value: function get(k) {
-      return this._cache[k];
+      return this._cache.get(k);
     }
   }, {
     key: "delete",
     value: function _delete(k) {
-      return delete this._cache[k];
+      this._cache.del(k);
     }
   }, {
     key: "add",
-    value: function add(newItems) {
-      this._cache = Object.assign(this._cache, newItems);
+    value: function add(items) {
+      for (var k in items) {
+        this._cache.set(k, items[k]);
+      }
     }
   }, {
     key: "length",
     get: function get() {
-      return Object.values(this._cache).length;
+      return this._cache.length;
     }
   }]);
   return EntryIndex;

lib/es5/log-io.js

@@ -53,6 +53,7 @@ var LogIO = /*#__PURE__*/function () {
       var _toMultihash = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee(ipfs, log) {
         var _ref,
             format,
+            values,
             _args = arguments;
 
         return _regenerator["default"].wrap(function _callee$(_context) {
@@ -78,20 +79,25 @@ var LogIO = /*#__PURE__*/function () {
 
               case 5:
                 if (!isDefined(format)) format = 'dag-cbor';
+                _context.next = 8;
+                return log.values();
 
-                if (!(log.values.length < 1)) {
-                  _context.next = 8;
+              case 8:
+                values = _context.sent;
+
+                if (!(values.length < 1)) {
+                  _context.next = 11;
                   break;
                 }
 
                 throw new Error('Can\'t serialize an empty log');
 
-              case 8:
+              case 11:
                 return _context.abrupt("return", io.write(ipfs, format, log.toJSON(), {
                   links: IPLD_LINKS
                 }));
 
-              case 9:
+              case 12:
               case "end":
                 return _context.stop();
             }

package.json

@@ -20,6 +20,7 @@
   ],
   "dependencies": {
     "json-stringify-deterministic": "^1.0.1",
+    "lru-cache": "^6.0.0",
     "multihashing-async": "^2.0.1",
     "orbit-db-identity-provider": "^0.3.1",
     "orbit-db-io": "^1.0.1",

src/entry-index.js

@@ -1,28 +1,33 @@
 'use strict'
 
+const LRU = require('lru-cache')
+
 class EntryIndex {
-  constructor (entries = {}) {
-    this._cache = entries
+  constructor (entries = {}, cacheSize = Infinity) {
+    this._cache = new LRU({ max: cacheSize })
+    this.add(entries)
   }
 
   set (k, v) {
-    this._cache[k] = v
+    this._cache.set(k, v)
   }
 
   get (k) {
-    return this._cache[k]
+    return this._cache.get(k)
   }
 
   delete (k) {
-    return delete this._cache[k]
+    this._cache.del(k)
   }
 
-  add (newItems) {
-    this._cache = Object.assign(this._cache, newItems)
+  add (items) {
+    for (const k in items) {
+      this._cache.set(k, items[k])
+    }
   }
 
   get length () {
-    return Object.values(this._cache).length
+    return this._cache.length
   }
 }
 

src/log-io.js

@@ -23,7 +23,9 @@ class LogIO {
     if (!isDefined(ipfs)) throw LogError.IPFSNotDefinedError()
     if (!isDefined(log)) throw LogError.LogNotDefinedError()
     if (!isDefined(format)) format = 'dag-cbor'
-    if (log.values.length < 1) throw new Error('Can\'t serialize an empty log')
+
+    const values = await log.values()
+    if (values.length < 1) throw new Error('Can\'t serialize an empty log')
 
     return io.write(ipfs, format, log.toJSON(), { links: IPLD_LINKS })
   }