Run results for: Lodash.isEqual vs object-hash vs JSON.stringify: Duplicate data detection for array of objects with nested properties and lots of records - Final

Run details:

User agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36

Browser: Chrome 123

Operating system: Windows

Device Platform: Desktop

Date tested: 2 years ago

Set + JSON.stringify + native SHA

3K/s

Set + JSON.stringify

144/s

Iteration + _.isEqual

54/s

Set + objectHash

40/s

Iteration + JSON.stringify

0/s

View exact numbers

Test name	Executions per second
✓ Set + JSON.stringify + native SHA	2,751 Ops/sec
Set + JSON.stringify	144 Ops/sec
Iteration + _.isEqual	54 Ops/sec
Set + objectHash	40 Ops/sec
Iteration + JSON.stringify	0 Ops/sec

HTML Preparation code:

<script src="https://cdn.jsdelivr.net/npm/object-hash@2.0.3/dist/object_hash.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/lodash@4.17.4/lodash.min.js"></script>

Script Preparation code:

async function H(m) {
  const msgUint8 = new TextEncoder().encode(m);
  const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
  const hashArray = Array.from(new Uint8Array(hashBuffer));
  return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
}

function JSONStringifyOrdered(obj) {
  const allKeys = new Set();
  JSON.stringify(obj, (key, value) => (allKeys.add(key), value));
  return JSON.stringify(obj, Array.from(allKeys).sort());
}

async function prepare() {
  window.NUM_ITEMS = 1000;
  window.NUM_TAKE = 20;
  window.input = [];
  window.data = [];
  window.indices = [...Array(window.NUM_ITEMS).keys()];

  // Populate data: entries with a nested array of 50 random entries
  for (const i of window.indices) {
    window.data[i] = {
      ids: window.indices
        .slice(0, 1000)
        .map((x) => Math.floor(Math.random() * window.NUM_ITEMS * 10)),
      counter: i,
      counter2: Math.floor(Math.random() * i),
    };
  }

  window.check = window.data.map((obj) => false);
  window.stringified = window.data.map(JSONStringifyOrdered);
  window.strings = new Set(window.stringified);
  // calculate hashes for each data object
  window.hashes = new Set(window.data.map(objectHash));
  window.stringsHashes = new Set(window.stringified.map(objectHash));

  window.shas = new Set(await Promise.all(
    window.stringified.map(async (item) => await H(item))
  ));

  window.TAKE_IDX = window.indices.reverse().slice(0, window.NUM_TAKE);
  // Uncomment to choose random indices
  //window.TAKE_IDX = window.TAKE_IDX.map((x) => Math.floor(Math.random() * window.NUM_ITEMS));

  // Populate input: copies of the data
  for (const i of window.TAKE_IDX) {
    window.input[i] = {
      ids: [...window.data[i].ids],
      counter: window.data[i].counter,
      counter2: window.data[i].counter2,
    };
  }
  
  console.log(window.TAKE_IDX);
  console.log("END:prepare");
}

console.log("START:prepare");
prepare();

Tests:

Iteration + _.isEqual
for (const i of window.TAKE_IDX) { window.check[i] = window.data.some((item) => _.isEqual(item, window.input[i])); }
Iteration + JSON.stringify
for (const i of window.TAKE_IDX) { window.check[i] = window.stringified.some((item) => item === JSONStringifyOrdered(window.input[i])); }
Set + objectHash
for (const i of window.TAKE_IDX) { window.check[i] = window.hashes.has(objectHash(window.input[i])); }
Set + JSON.stringify
for (const i of window.TAKE_IDX) { window.check[i] = window.strings.has(JSONStringifyOrdered(window.input[i])); }
Set + JSON.stringify + native SHA
async function testNativeHash() { for (const i of window.TAKE_IDX) { window.check[i] = window.shas.has(await H(JSONStringifyOrdered(window.input[i]))); } } testNativeHash();