Gun reports all documents always not only changed ones

[koh-osug]

When I stop my node clients and restart it all documents are reported to the node, not just updated ones.

• The question is how can I differentiate between really new items and old items? E.g. if the list has 100 items the reported item list will be huge. Since I'm mirroring this into a classical database the initial load after starting my node would be actually to rewrite the whole database.

Example I add while my node client is not running I add:

{"docKey":"12345678789","counter":11}
Object { foo: "bar", counter: 12 }
​counter: 12
​foo: "bar"
​}

In the client although it has received already all previous event until counter 11 I still see:

getting a new document {
  _: {
    '#': 'testkey',
    '>': { counter: 1705088678010, foo: 1705088678010 }
  },
  counter: 3,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":0}',
    '>': { counter: 1705088820172, foo: 1705088820172 }
  },
  counter: 1,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":10}',
    '>': { counter: 1705089016708, foo: 1705089016708 }
  },
  counter: 11,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":11}',
    '>': { counter: 1705089098084, foo: 1705089098084 }
  },
  counter: 12,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":12}',
    '>': { counter: 1705089299089, foo: 1705089299089 }
  },
  counter: 13,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":1}',
    '>': { counter: 1705088827644, foo: 1705088827644 }
  },
  counter: 2,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":2}',
    '>': { counter: 1705088829660, foo: 1705088829660 }
  },
  counter: 3,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":3}',
    '>': { counter: 1705088911366, foo: 1705088911366 }
  },
  counter: 4,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":4}',
    '>': { counter: 1705088912461, foo: 1705088912461 }
  },
  counter: 5,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":5}',
    '>': { counter: 1705088914236, foo: 1705088914236 }
  },
  counter: 6,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":6}',
    '>': { counter: 1705088929828, foo: 1705088929828 }
  },
  counter: 7,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":7}',
    '>': { counter: 1705089012617, foo: 1705089012617 }
  },
  counter: 8,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":8}',
    '>': { counter: 1705089015268, foo: 1705089015268 }
  },
  counter: 9,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":9}',
    '>': { counter: 1705089016028, foo: 1705089016028 }
  },
  counter: 10,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': 'testkey',
    '>': { counter: 1705088678010, foo: 1705088678010 }
  },
  counter: 3,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":0}',
    '>': { counter: 1705088820172, foo: 1705088820172 }
  },
  counter: 1,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":10}',
    '>': { counter: 1705089016708, foo: 1705089016708 }
  },
  counter: 11,
  foo: 'bar'
}
getting a new document {
  _: {
    '#': '{"docKey":"12345678789","counter":11}',
    '>': { counter: 1705089098084, foo: 1705089098084 }
  },
  counter: 12,
  foo: 'bar'
}
...

My test.html creating data:

</style>
<div>
<input id="collection" readonly placeholder="collection" value="testdata">
</input>
</div>
<div>
<textarea id="key" rows="5" placeholder="document key">
    {
        "docKey": "12345678789"
    }
</textarea>
</div>
<div>
<textarea id="document"  rows="25" placeholder="document">
    {
      "foo": "bar"
    }
</textarea>
</div>
<button onclick="save()">Save</button>
<script src="https://cdn.jsdelivr.net/npm/gun/gun.js"></script><script>
    gun = Gun(['http://localhost:9999/gun']);
    let counter = 0
    function save() {
        const doc = document.getElementById('document').value;
        const collection = document.getElementById('collection').value;
        let key = document.getElementById('key').value;
        let _key = JSON.parse(key)
        _key.counter = counter++;
        key = JSON.stringify(_key)
        console.log("add key", key)
        let _doc = JSON.parse(doc)
        _doc.counter = counter
        let ref = gun.get(key).put(_doc)
        console.log("doc", _doc)
        gun.get(collection).set(ref)
    }

</script>

Open this in a browser.

My Gun proxy server.js:

import Gun from 'gun'
import {createServer} from "http";


async function run () {
    const server = createServer().listen(9999, "0.0.0.0");
    const gun = Gun({web: server});
}

await run()

Start with:

npm init es6 -y
npm install gun
node server.js

In the same dir my test peer test.js running inside node:

import Gun from 'gun'

async function run () {
    const gun = Gun('http://localhost:9999/gun');
    gun.get('testdata').map().on(function(data, key) {
        console.info(`getting a new document`, data)
    }, {
        change: false
    })
}

await run()

Start with:

node test.js