From 90dd03a303f145dd823fe758259ba9d5f3aa0bfa Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 30 Jul 2020 16:41:22 +0100 Subject: [PATCH 1/2] fix: replace node buffers with uint8arrays BREAKING CHANGES: - node `Buffer`s have been replaced with `Uint8Array`s --- migrations/migration-9/index.js | 2 +- migrations/migration-9/pin-set.js | 41 +++++++++++------------ package.json | 16 ++++----- src/repo/version.js | 4 +-- test/browser.js | 5 ++- test/init-test.js | 6 ++-- test/migrations/migration-9-test.js | 9 ++--- test/test-migrations/migration-2/index.js | 6 ++-- test/version-test.js | 11 +++--- 9 files changed, 48 insertions(+), 52 deletions(-) diff --git a/migrations/migration-9/index.js b/migrations/migration-9/index.js index ff87479..5cbafcd 100644 --- a/migrations/migration-9/index.js +++ b/migrations/migration-9/index.js @@ -66,7 +66,7 @@ async function pinsToDAG (blockstore, datastore, pinstore) { } } - const pinRoot = new dagpb.DAGNode(Buffer.alloc(0), [ + const pinRoot = new dagpb.DAGNode(new Uint8Array(), [ await pinset.storeSet(blockstore, PinTypes.recursive, recursivePins), await pinset.storeSet(blockstore, PinTypes.direct, directPins) ]) diff --git a/migrations/migration-9/pin-set.js b/migrations/migration-9/pin-set.js index a3f25c3..568dbaf 100644 --- a/migrations/migration-9/pin-set.js +++ b/migrations/migration-9/pin-set.js @@ -8,8 +8,11 @@ const dagpb = require('ipld-dag-pb') const { DAGNode, DAGLink } = dagpb const multicodec = require('multicodec') const pbSchema = require('./pin.proto') -const { Buffer } = require('buffer') const { cidToKey, DEFAULT_FANOUT, MAX_ITEMS, EMPTY_KEY } = require('./utils') +const uint8ArrayConcat = require('ipfs-utils/src/uint8arrays/concat') +const uint8ArrayCompare = require('ipfs-utils/src/uint8arrays/compare') +const uint8ArrayToString = require('ipfs-utils/src/uint8arrays/to-string') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const pb = protobuf(pbSchema) @@ -50,12 +53,13 @@ function readHeader (rootNode) { } function hash (seed, key) { - const buf = Buffer.alloc(4) - buf.writeUInt32LE(seed, 0) - const data = Buffer.concat([ - buf, Buffer.from(toB58String(key)) - ]) - return fnv1a(data.toString('binary')) + const buffer = new ArrayBuffer(4) + const dataView = new DataView(buffer) + dataView.setUint32(0, seed, true) + const encodedKey = uint8ArrayFromString(toB58String(key)) + const data = uint8ArrayConcat([buf, encodedKey], buf.length + encodedKey.length) + + return fnv1a(uint8ArrayToString(data)) } async function * walkItems (blockstore, node) { @@ -106,9 +110,9 @@ function storeItems (blockstore, items) { fanout: DEFAULT_FANOUT, seed: depth }) - const headerBuf = Buffer.concat([ - Buffer.from(varint.encode(pbHeader.length)), pbHeader - ]) + + const header = varint.encode(pbHeader.length) + const headerBuf = uint8ArrayConcat([header, pbHeader]) const fanoutLinks = [] for (let i = 0; i < DEFAULT_FANOUT; i++) { @@ -120,16 +124,16 @@ function storeItems (blockstore, items) { .map(item => { return ({ link: new DAGLink('', 1, item.key), - data: item.data || Buffer.alloc(0) + data: item.data || new Uint8Array() }) }) // sorting makes any ordering of `pins` produce the same DAGNode - .sort((a, b) => Buffer.compare(a.link.Hash.buffer, b.link.Hash.buffer)) + .sort((a, b) => { + return uint8ArrayCompare(a.link.Hash.buffer, b.link.Hash.buffer) + }) const rootLinks = fanoutLinks.concat(nodes.map(item => item.link)) - const rootData = Buffer.concat( - [headerBuf].concat(nodes.map(item => item.data)) - ) + const rootData = uint8ArrayConcat([headerBuf, ...nodes.map(item => item.data)]) return new DAGNode(rootData, rootLinks) } else { @@ -162,13 +166,6 @@ function storeItems (blockstore, items) { } async function storeChild (child, binIdx) { - const opts = { - version: 0, - format: multicodec.DAG_PB, - hashAlg: multicodec.SHA2_256, - preload: false - } - const buf = dagpb.util.serialize(child) const cid = dagpb.util.cid(buf, { cidVersion: 0, diff --git a/package.json b/package.json index 5c78e62..7fc48df 100644 --- a/package.json +++ b/package.json @@ -44,18 +44,18 @@ "docs": "aegir docs" }, "dependencies": { - "buffer": "^5.6.0", "cbor": "^5.0.2", "chalk": "^4.0.0", - "cids": "^0.8.3", - "datastore-core": "^1.1.0", - "datastore-fs": "^1.0.0", - "datastore-level": "^1.1.0", + "cids": "multiformats/js-cid#fix/support-uint8arrays", + "datastore-core": "^2.0.0", + "datastore-fs": "^2.0.0", + "datastore-level": "^2.0.0", "debug": "^4.1.0", "fnv1a": "^1.0.1", - "interface-datastore": "^1.0.2", - "ipld-dag-pb": "^0.18.5", - "multibase": "^1.0.1", + "interface-datastore": "^2.0.0", + "ipfs-utils": "ipfs/js-ipfs-utils#feat/add-utility-uint8array-functions", + "ipld-dag-pb": "ipld/js-ipld-dag-pb#fix/replace-node-buffers-with-unit8arrays", + "multibase": "^2.0.0", "multicodec": "^1.0.3", "multihashing-async": "^1.0.0", "proper-lockfile": "^4.1.1", diff --git a/src/repo/version.js b/src/repo/version.js index 34ec0ff..92fa7fa 100644 --- a/src/repo/version.js +++ b/src/repo/version.js @@ -1,9 +1,9 @@ 'use strict' -const { Buffer } = require('buffer') const errors = require('../errors') const repoInit = require('./init') const Datastore = require('datastore-fs') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const Key = require('interface-datastore').Key @@ -43,7 +43,7 @@ async function getVersion (path) { async function setVersion (path, version) { const store = new Datastore(path, { extension: '', createIfMissing: false }) await store.open() - await store.put(versionKey, Buffer.from(String(version))) + await store.put(versionKey, uint8ArrayFromString(String(version))) await store.close() } diff --git a/test/browser.js b/test/browser.js index 5ea35cd..83a3a8e 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const loadFixture = require('aegir/fixtures') const Datastore = require('datastore-level') @@ -24,8 +23,8 @@ async function createAndLoadRepo () { const store = new Datastore(dir, { extension: '', createIfMissing: true }) await store.open() - await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) - await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) + await store.put(VERSION_KEY, loadFixture('test/fixtures/test-repo/version')) + await store.put(CONFIG_KEY, loadFixture('test/fixtures/test-repo/config')) return dir } diff --git a/test/init-test.js b/test/init-test.js index 0ceb63d..341f769 100644 --- a/test/init-test.js +++ b/test/init-test.js @@ -1,12 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { expect } = require('./util') const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const repoInit = require('../src/repo/init') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') module.exports = (setup, cleanup) => { let dir @@ -23,8 +23,8 @@ module.exports = (setup, cleanup) => { const configKey = new Key('config') const store = new Datastore(dir, { extension: '', createIfMissing: false }) await store.open() - await store.put(versionKey, Buffer.from('7')) - await store.put(configKey, Buffer.from('config')) + await store.put(versionKey, uint8ArrayFromString('7')) + await store.put(configKey, uint8ArrayFromString('config')) await store.close() expect(await repoInit.isRepoInitialized(dir)).to.be.true() diff --git a/test/migrations/migration-9-test.js b/test/migrations/migration-9-test.js index cb36644..b9f2c37 100644 --- a/test/migrations/migration-9-test.js +++ b/test/migrations/migration-9-test.js @@ -12,13 +12,14 @@ const multicodec = require('multicodec') const multibase = require('multibase') const all = require('it-all') const cbor = require('cbor') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const migration = require('../../migrations/migration-9') const { createStore, cidToKey, PIN_DS_KEY, DEFAULT_FANOUT } = require('../../migrations/migration-9/utils') const CID = require('cids') function keyToCid (key) { - const buf = Buffer.from(multibase.encoding('base32upper').decode(key.toString().split('/').pop())) + const buf = multibase.encoding('base32upper').decode(key.toString().split('/').pop()) return new CID(buf) } @@ -49,11 +50,11 @@ async function bootstrapBlocks (blockstore, datastore) { ) const bucket = new Array(DEFAULT_FANOUT).fill(0).map(() => new DAGLink('', 1, emptyBlock.Hash)) const directLinks = await putNode( - new DAGNode(Buffer.from('CggBEIACHQAAAAA=', 'base64'), bucket), + new DAGNode(uint8ArrayFromString('CggBEIACHQAAAAA=', 'base64urlpad'), bucket), 'QmbxHkprr5qdLSK8EZWdBzKFzNXGoKrxb7A4PHX3eH6JPp' ) const recursiveLinks = await putNode( - new DAGNode(Buffer.from('CggBEIACHQAAAAA=', 'base64'), [ + new DAGNode(uint8ArrayFromString('CggBEIACHQAAAAA=', 'base64urlpad'), [ ...bucket, new DAGLink('', 1, pinnedCid) ]), @@ -61,7 +62,7 @@ async function bootstrapBlocks (blockstore, datastore) { ) const pinRoot = await putNode( - new DAGNode(Buffer.alloc(0), [ + new DAGNode(new Uint8Array(), [ new DAGLink('direct', directLinks.Tsize, directLinks.Hash), new DAGLink('recursive', recursiveLinks.Tsize, recursiveLinks.Hash) ]), diff --git a/test/test-migrations/migration-2/index.js b/test/test-migrations/migration-2/index.js index a5cefce..cfcaeac 100644 --- a/test/test-migrations/migration-2/index.js +++ b/test/test-migrations/migration-2/index.js @@ -1,9 +1,9 @@ 'use strict' -const { Buffer } = require('buffer') const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const _set = require('just-safe-set') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const CONFIG_KEY = new Key('config') const NEW_API_ADDRESS = '/ip6/::/tcp/5001' @@ -89,7 +89,7 @@ async function migrate (repoPath, options, isBrowser) { // Modify allowed origin _set(config, 'Gateway.HTTPHeaders.Access-Control-Allow-Origin', 'some.origin.com') - const buf = Buffer.from(JSON.stringify(config, null, 2)) + const buf = uint8ArrayFromString(JSON.stringify(config, null, 2)) await store.put(CONFIG_KEY, buf) } finally { await store.close() @@ -109,7 +109,7 @@ async function revert (repoPath, options, isBrowser) { // Reset origin _set(config, 'Gateway.HTTPHeaders.Access-Control-Allow-Origin', '*') - const buf = Buffer.from(JSON.stringify(config, null, 2)) + const buf = uint8ArrayFromString(JSON.stringify(config, null, 2)) await store.put(CONFIG_KEY, buf) } finally { await store.close() diff --git a/test/version-test.js b/test/version-test.js index 20aac49..ba75665 100644 --- a/test/version-test.js +++ b/test/version-test.js @@ -1,13 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { expect } = require('./util') const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const version = require('../src/repo/version') - +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const errors = require('../src/errors') // When new versioning mechanism is introduced in new version don't forget to update @@ -34,8 +33,8 @@ module.exports = (setup, cleanup) => { // Create version file const store = new Datastore(dir, { extension: '', createIfMissing: false }) await store.open() - await store.put(new Key('config'), Buffer.from('some dummy config')) - await store.put(new Key('version'), Buffer.from('7')) + await store.put(new Key('config'), uint8ArrayFromString('some dummy config')) + await store.put(new Key('version'), uint8ArrayFromString('7')) await store.close() expect(await version.getVersion(dir)).to.be.equal(7) @@ -47,8 +46,8 @@ module.exports = (setup, cleanup) => { // Create version file const store = new Datastore(dir, { extension: '', createIfMissing: false }) await store.open() - await store.put(new Key('config'), Buffer.from('some dummy config')) - await store.put(new Key('version'), Buffer.from('5')) + await store.put(new Key('config'), uint8ArrayFromString('some dummy config')) + await store.put(new Key('version'), uint8ArrayFromString('5')) await store.close() await version.setVersion(dir, 7) From 5cc9eb9129df9029353a77fee1d276d9a654f6fb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 5 Aug 2020 07:27:28 +0100 Subject: [PATCH 2/2] chore: update gh dependencies --- migrations/migration-8/index.js | 7 ++++--- migrations/migration-9/pin-set.js | 13 +++++++------ package.json | 14 +++++++------- src/repo/version.js | 2 +- test/init-test.js | 2 +- test/migrations/migration-9-test.js | 2 +- test/test-migrations/migration-2/index.js | 2 +- test/version-test.js | 2 +- 8 files changed, 23 insertions(+), 21 deletions(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 803544b..50ee69b 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -7,6 +7,7 @@ const ShardingStore = core.ShardingDatastore const mb = require('multibase') const utils = require('../../src/utils') const log = require('debug')('ipfs-repo-migrations:migration-8') +const uint8ArrayToString = require('uint8arrays/to-string') // This function in js-ipfs-repo defaults to not using sharding // but the default value of the options.sharding is true hence this @@ -31,7 +32,7 @@ function keyToMultihash (key) { multihash = mb.encode('base32', multihash).slice(1) // Should be uppercase for interop with go - multihash = multihash.toString().toUpperCase() + multihash = uint8ArrayToString(multihash).toUpperCase() return new Key(`/${multihash}`, false) } @@ -40,9 +41,9 @@ function keyToCid (key) { const buf = mb.decode(`b${key.toString().slice(1)}`) // CID to Key - const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1) + const multihash = mb.encode('base32', new CID(1, 'raw', buf).bytes).slice(1) - return new Key(`/${multihash}`.toUpperCase(), false) + return new Key(`/${uint8ArrayToString(multihash)}`.toUpperCase(), false) } async function process (repoPath, options, keyFunction){ diff --git a/migrations/migration-9/pin-set.js b/migrations/migration-9/pin-set.js index 568dbaf..aa46b9a 100644 --- a/migrations/migration-9/pin-set.js +++ b/migrations/migration-9/pin-set.js @@ -9,10 +9,11 @@ const { DAGNode, DAGLink } = dagpb const multicodec = require('multicodec') const pbSchema = require('./pin.proto') const { cidToKey, DEFAULT_FANOUT, MAX_ITEMS, EMPTY_KEY } = require('./utils') -const uint8ArrayConcat = require('ipfs-utils/src/uint8arrays/concat') -const uint8ArrayCompare = require('ipfs-utils/src/uint8arrays/compare') -const uint8ArrayToString = require('ipfs-utils/src/uint8arrays/to-string') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayCompare = require('uint8arrays/compare') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayEquals = require('uint8arrays/equals') const pb = protobuf(pbSchema) @@ -72,7 +73,7 @@ async function * walkItems (blockstore, node) { // if a fanout bin is not 'empty', dig into and walk its DAGLinks const linkHash = link.Hash - if (!EMPTY_KEY.equals(linkHash.buffer)) { + if (!uint8ArrayEquals(EMPTY_KEY, linkHash.bytes)) { // walk the links of this fanout bin const buf = await blockstore.get(cidToKey(linkHash)) const node = dagpb.util.deserialize(buf) @@ -129,7 +130,7 @@ function storeItems (blockstore, items) { }) // sorting makes any ordering of `pins` produce the same DAGNode .sort((a, b) => { - return uint8ArrayCompare(a.link.Hash.buffer, b.link.Hash.buffer) + return uint8ArrayCompare(a.link.Hash.bytes, b.link.Hash.bytes) }) const rootLinks = fanoutLinks.concat(nodes.map(item => item.link)) diff --git a/package.json b/package.json index 7fc48df..d262aed 100644 --- a/package.json +++ b/package.json @@ -46,20 +46,20 @@ "dependencies": { "cbor": "^5.0.2", "chalk": "^4.0.0", - "cids": "multiformats/js-cid#fix/support-uint8arrays", + "cids": "^1.0.0", "datastore-core": "^2.0.0", "datastore-fs": "^2.0.0", "datastore-level": "^2.0.0", "debug": "^4.1.0", "fnv1a": "^1.0.1", "interface-datastore": "^2.0.0", - "ipfs-utils": "ipfs/js-ipfs-utils#feat/add-utility-uint8array-functions", - "ipld-dag-pb": "ipld/js-ipld-dag-pb#fix/replace-node-buffers-with-unit8arrays", - "multibase": "^2.0.0", - "multicodec": "^1.0.3", - "multihashing-async": "^1.0.0", + "ipld-dag-pb": "^0.20.0", + "multibase": "^3.0.0", + "multicodec": "^2.0.0", + "multihashing-async": "^2.0.0", "proper-lockfile": "^4.1.1", - "protons": "^1.2.1", + "protons": "^2.0.0", + "uint8arrays": "^1.0.0", "varint": "^5.0.0", "yargs": "^15.3.1", "yargs-promise": "^1.1.0" diff --git a/src/repo/version.js b/src/repo/version.js index 92fa7fa..621f09f 100644 --- a/src/repo/version.js +++ b/src/repo/version.js @@ -3,7 +3,7 @@ const errors = require('../errors') const repoInit = require('./init') const Datastore = require('datastore-fs') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const Key = require('interface-datastore').Key diff --git a/test/init-test.js b/test/init-test.js index 341f769..6c8d534 100644 --- a/test/init-test.js +++ b/test/init-test.js @@ -6,7 +6,7 @@ const { expect } = require('./util') const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const repoInit = require('../src/repo/init') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = (setup, cleanup) => { let dir diff --git a/test/migrations/migration-9-test.js b/test/migrations/migration-9-test.js index b9f2c37..0355ab7 100644 --- a/test/migrations/migration-9-test.js +++ b/test/migrations/migration-9-test.js @@ -12,7 +12,7 @@ const multicodec = require('multicodec') const multibase = require('multibase') const all = require('it-all') const cbor = require('cbor') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const migration = require('../../migrations/migration-9') const { createStore, cidToKey, PIN_DS_KEY, DEFAULT_FANOUT } = require('../../migrations/migration-9/utils') diff --git a/test/test-migrations/migration-2/index.js b/test/test-migrations/migration-2/index.js index cfcaeac..4d8aa0b 100644 --- a/test/test-migrations/migration-2/index.js +++ b/test/test-migrations/migration-2/index.js @@ -3,7 +3,7 @@ const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const _set = require('just-safe-set') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const CONFIG_KEY = new Key('config') const NEW_API_ADDRESS = '/ip6/::/tcp/5001' diff --git a/test/version-test.js b/test/version-test.js index ba75665..e4613b5 100644 --- a/test/version-test.js +++ b/test/version-test.js @@ -6,7 +6,7 @@ const { expect } = require('./util') const Datastore = require('datastore-fs') const Key = require('interface-datastore').Key const version = require('../src/repo/version') -const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const errors = require('../src/errors') // When new versioning mechanism is introduced in new version don't forget to update