diff --git a/README.md b/README.md index 6dbe65b382..541eca795c 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,6 @@ You can check the development status at the [Kanban Board](https://waffle.io/ipf [![Throughput Graph](https://graphs.waffle.io/ipfs/js-ipfs/throughput.svg)](https://waffle.io/ipfs/js-ipfs/metrics/throughput) -**Please read this:** DHT (automatic content discovery) and Circuit Relay (pierce through NATs and dial between any node in the network) are two fundamental pieces that are not finalized yet. There are multiple applications that can be built without these two services but nevertheless they are fundamental to get that magic IPFS experience. If you want to track progress or contribute, please follow: - -- DHT: https://github.com/ipfs/js-ipfs/pull/856 -- ✅ Relay: https://github.com/ipfs/js-ipfs/pull/1063 - [**`Weekly Core Dev Calls`**](https://github.com/ipfs/pm/issues/650) ## Tech Lead @@ -319,7 +314,6 @@ Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `ipnsPubsub` (boolean): Enable pub-sub on IPNS. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) -- `dht` (boolean): Enable KadDHT. **This is currently not interoperable with `go-ipfs`.** ##### `options.config` @@ -600,7 +594,13 @@ The core API is grouped into several areas: - [`ipfs.bootstrap.add(addr, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstrapadd) - [`ipfs.bootstrap.rm(peer, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstraprm) -- dht (not implemented yet) +- [dht](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/) + - [`ipfs.dht.findPeer(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindpeer) + - [`ipfs.dht.findProvs(multihash, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindprovs) + - [`ipfs.dht.get(key, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtget) + - [`ipfs.dht.provide(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtprovide) + - [`ipfs.dht.put(key, value, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtput) + - [`ipfs.dht.query(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtquery) - [pubsub](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md) - [`ipfs.pubsub.subscribe(topic, handler, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubsubscribe) diff --git a/package.json b/package.json index 0ca90a57af..eacbfc1937 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "form-data": "^2.3.3", "hat": "0.0.3", "interface-ipfs-core": "~0.96.0", - "ipfsd-ctl": "~0.40.1", + "ipfsd-ctl": "~0.41.0", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", "qs": "^6.5.2", @@ -129,7 +129,7 @@ "joi": "^14.3.0", "joi-browser": "^13.4.0", "joi-multiaddr": "^4.0.0", - "libp2p": "~0.24.1", + "libp2p": "~0.25.0-rc.0", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.16.0", "libp2p-kad-dht": "~0.14.4", @@ -141,7 +141,7 @@ "libp2p-tcp": "~0.13.0", "libp2p-webrtc-star": "~0.15.5", "libp2p-websocket-star-multi": "~0.4.0", - "libp2p-websockets": "~0.12.0", + "libp2p-websockets": "~0.12.2", "lodash": "^4.17.11", "mafmt": "^6.0.2", "mime-types": "^2.1.21", diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index 8afcdbe3d6..ba8dbd642a 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -18,10 +18,6 @@ module.exports = { type: 'boolean', default: false }) - .option('enable-dht-experiment', { - type: 'boolean', - default: false - }) .option('offline', { desc: 'Run offline. Do not connect to the rest of the network but provide local API.', default: false diff --git a/src/cli/commands/dht.js b/src/cli/commands/dht.js new file mode 100644 index 0000000000..d899cc3c5c --- /dev/null +++ b/src/cli/commands/dht.js @@ -0,0 +1,14 @@ +'use strict' + +module.exports = { + command: 'dht ', + + description: 'Issue commands directly through the DHT.', + + builder (yargs) { + return yargs.commandDir('dht') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/dht/find-peer.js b/src/cli/commands/dht/find-peer.js new file mode 100644 index 0000000000..933919335f --- /dev/null +++ b/src/cli/commands/dht/find-peer.js @@ -0,0 +1,23 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'findpeer ', + + describe: 'Find the multiaddresses associated with a Peer ID.', + + builder: {}, + + handler ({ getIpfs, peerID, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + const peers = await ipfs.dht.findPeer(peerID) + const addresses = peers.multiaddrs.toArray().map((ma) => ma.toString()) + + addresses.forEach((addr) => { + print(addr) + }) + })()) + } +} diff --git a/src/cli/commands/dht/find-providers.js b/src/cli/commands/dht/find-providers.js new file mode 100644 index 0000000000..46ec3bc90d --- /dev/null +++ b/src/cli/commands/dht/find-providers.js @@ -0,0 +1,33 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'findprovs ', + + describe: 'Find peers that can provide a specific value, given a key.', + + builder: { + 'num-providers': { + alias: 'n', + describe: 'The number of providers to find. Default: 20.', + default: 20 + } + }, + + handler (argv) { + const { getIpfs, key, resolve } = argv + const opts = { + maxNumProviders: argv['num-providers'] + } + + resolve((async () => { + const ipfs = await getIpfs() + const provs = await ipfs.dht.findProvs(key, opts) + + provs.forEach((element) => { + print(element.id.toB58String()) + }) + })()) + } +} diff --git a/src/cli/commands/dht/get.js b/src/cli/commands/dht/get.js new file mode 100644 index 0000000000..b2ba34b3fe --- /dev/null +++ b/src/cli/commands/dht/get.js @@ -0,0 +1,20 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'get ', + + describe: 'Given a key, query the routing system for its best value.', + + builder: {}, + + handler ({ getIpfs, key, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + const value = await ipfs.dht.get(key) + + print(value) + })()) + } +} diff --git a/src/cli/commands/dht/provide.js b/src/cli/commands/dht/provide.js new file mode 100644 index 0000000000..a787f5aa95 --- /dev/null +++ b/src/cli/commands/dht/provide.js @@ -0,0 +1,26 @@ +'use strict' + +module.exports = { + command: 'provide ', + + describe: 'Announce to the network that you are providing given values.', + + builder: { + recursive: { + alias: 'r', + recursive: 'Recursively provide entire graph.', + default: false + } + }, + + handler ({ getIpfs, key, recursive, resolve }) { + const opts = { + recursive + } + + resolve((async () => { + const ipfs = await getIpfs() + await ipfs.dht.provide(key, opts) + })()) + } +} diff --git a/src/cli/commands/dht/put.js b/src/cli/commands/dht/put.js new file mode 100644 index 0000000000..068e077eee --- /dev/null +++ b/src/cli/commands/dht/put.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = { + command: 'put ', + + describe: 'Write a key/value pair to the routing system.', + + builder: {}, + + handler ({ getIpfs, key, value, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + await ipfs.dht.put(key, value) + })()) + } +} diff --git a/src/cli/commands/dht/query.js b/src/cli/commands/dht/query.js new file mode 100644 index 0000000000..dd5e716257 --- /dev/null +++ b/src/cli/commands/dht/query.js @@ -0,0 +1,22 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'query ', + + describe: 'Find the closest Peer IDs to a given Peer ID by querying the DHT.', + + builder: {}, + + handler ({ getIpfs, peerID, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + const result = await ipfs.dht.query(peerID) + + result.forEach((peerID) => { + print(peerID.id.toB58String()) + }) + })()) + } +} diff --git a/src/core/components/dht.js b/src/core/components/dht.js index a4901bdfce..e5fc62dbec 100644 --- a/src/core/components/dht.js +++ b/src/core/components/dht.js @@ -3,11 +3,16 @@ const promisify = require('promisify-es6') const every = require('async/every') const PeerId = require('peer-id') +const PeerInfo = require('peer-info') const CID = require('cids') const each = require('async/each') -const setImmediate = require('async/setImmediate') -// const bsplit = require('buffer-split') -const errCode = require('err-code') +const nextTick = require('async/nextTick') + +const errcode = require('err-code') + +const debug = require('debug') +const log = debug('jsipfs:dht') +log.error = debug('jsipfs:dht:error') module.exports = (self) => { return { @@ -15,14 +20,12 @@ module.exports = (self) => { * Given a key, query the DHT for its best value. * * @param {Buffer} key + * @param {Object} options - get options + * @param {number} options.timeout - optional timeout * @param {function(Error)} [callback] * @returns {Promise|void} */ get: promisify((key, options, callback) => { - if (!Buffer.isBuffer(key)) { - return callback(new Error('Not valid key')) - } - if (typeof options === 'function') { callback = options options = {} @@ -30,7 +33,17 @@ module.exports = (self) => { options = options || {} - self.libp2p.dht.get(key, options.timeout, callback) + if (!Buffer.isBuffer(key)) { + try { + key = (new CID(key)).buffer + } catch (err) { + log.error(err) + + return nextTick(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } + } + + self.libp2p.dht.get(key, options, callback) }), /** @@ -47,7 +60,13 @@ module.exports = (self) => { */ put: promisify((key, value, callback) => { if (!Buffer.isBuffer(key)) { - return callback(new Error('Not valid key')) + try { + key = (new CID(key)).buffer + } catch (err) { + log.error(err) + + return nextTick(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } } self.libp2p.dht.put(key, value, callback) @@ -57,72 +76,53 @@ module.exports = (self) => { * Find peers in the DHT that can provide a specific value, given a key. * * @param {CID} key - They key to find providers for. + * @param {Object} options - findProviders options + * @param {number} options.timeout - how long the query should maximally run, in milliseconds (default: 60000) + * @param {number} options.maxNumProviders - maximum number of providers to find * @param {function(Error, Array)} [callback] * @returns {Promise|void} */ - findprovs: promisify((key, opts, callback) => { - if (typeof opts === 'function') { - callback = opts - opts = {} + findProvs: promisify((key, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } - opts = opts || {} + options = options || {} if (typeof key === 'string') { try { key = new CID(key) } catch (err) { - return setImmediate(() => callback(errCode(err, 'ERR_INVALID_CID'))) - } - } + log.error(err) - if (typeof opts === 'function') { - callback = opts - opts = {} + return nextTick(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } } - opts = opts || {} - - self.libp2p.contentRouting.findProviders(key, opts.timeout || null, callback) + self.libp2p.contentRouting.findProviders(key, options, callback) }), /** * Query the DHT for all multiaddresses associated with a `PeerId`. * * @param {PeerId} peer - The id of the peer to search for. - * @param {function(Error, Array)} [callback] - * @returns {Promise>|void} + * @param {function(Error, PeerInfo)} [callback] + * @returns {Promise|void} */ - findpeer: promisify((peer, callback) => { + findPeer: promisify((peer, callback) => { if (typeof peer === 'string') { peer = PeerId.createFromB58String(peer) } - self.libp2p.peerRouting.findPeer(peer, (err, info) => { - if (err) { - return callback(err) - } - - // convert to go-ipfs return value, we need to revisit - // this. For now will just conform. - const goResult = [ - { - Responses: [{ - ID: info.id.toB58String(), - Addresses: info.multiaddrs.toArray().map((a) => a.toString()) - }] - } - ] - - callback(null, goResult) - }) + self.libp2p.peerRouting.findPeer(peer, callback) }), /** * Announce to the network that we are providing given values. * * @param {CID|Array} keys - The keys that should be announced. - * @param {Object} [options={}] + * @param {Object} options - provide options * @param {bool} [options.recursive=false] - Provide not only the given object but also all objects linked from it. * @param {function(Error)} [callback] * @returns {Promise|void} @@ -147,11 +147,15 @@ module.exports = (self) => { } if (!has) { - return callback(new Error('block(s) not found locally, cannot provide')) + const errMsg = 'block(s) not found locally, cannot provide' + + log.error(errMsg) + return callback(errcode(errMsg, 'ERR_BLOCK_NOT_FOUND')) } if (options.recursive) { // TODO: Implement recursive providing + return callback(errcode('not implemented yet', 'ERR_NOT_IMPLEMENTED_YET')) } else { each(keys, (cid, cb) => { self.libp2p.contentRouting.provide(cid, cb) @@ -164,22 +168,27 @@ module.exports = (self) => { * Find the closest peers to a given `PeerId`, by querying the DHT. * * @param {PeerId} peer - The `PeerId` to run the query agains. - * @param {function(Error, Array)} [callback] - * @returns {Promise>|void} + * @param {function(Error, Array)} [callback] + * @returns {Promise>|void} */ query: promisify((peerId, callback) => { if (typeof peerId === 'string') { - peerId = PeerId.createFromB58String(peerId) + try { + peerId = PeerId.createFromB58String(peerId) + } catch (err) { + log.error(err) + return callback(err) + } } // TODO expose this method in peerRouting self.libp2p._dht.getClosestPeers(peerId.toBytes(), (err, peerIds) => { if (err) { + log.error(err) return callback(err) } - callback(null, peerIds.map((id) => { - return { ID: id.toB58String() } - })) + + callback(null, peerIds.map((id) => new PeerInfo(id))) }) }) } diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index bf404bf1a0..c45b65f625 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -76,6 +76,11 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { } }, dht: { + kBucketSize: get(options, 'dht.kBucketSize', 20), + enabled: get(options, 'dht.enabled', true) && !(get(options, 'offline', false)), + randomWalk: { + enabled: get(options, 'dht.randomWalk.enabled', true) + }, validators: { ipns: ipnsUtils.validator }, @@ -84,7 +89,6 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { } }, EXPERIMENTAL: { - dht: get(options, 'EXPERIMENTAL.dht', false), pubsub: get(options, 'EXPERIMENTAL.pubsub', false) } }, diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index 91f72bf90b..31ecfffdf0 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -6,8 +6,8 @@ const protobuf = require('protons') const fnv1a = require('fnv1a') const varint = require('varint') const { DAGNode, DAGLink } = require('ipld-dag-pb') -const some = require('async/some') -const eachOf = require('async/eachOf') +const someSeries = require('async/someSeries') +const eachOfSeries = require('async/eachOfSeries') const pbSchema = require('./pin.proto') @@ -69,7 +69,7 @@ exports = module.exports = function (dag) { return searchChildren(root, callback) function searchChildren (root, cb) { - some(root.links, ({ cid }, done) => { + someSeries(root.links, ({ cid }, done) => { const bs58Link = toB58String(cid) if (bs58Link === childhash) { @@ -174,7 +174,7 @@ exports = module.exports = function (dag) { return bins }, {}) - eachOf(bins, (bin, idx, eachCb) => { + eachOfSeries(bins, (bin, idx, eachCb) => { storePins( bin, depth + 1, @@ -233,7 +233,7 @@ exports = module.exports = function (dag) { return callback(err) } - eachOf(node.links, (link, idx, eachCb) => { + eachOfSeries(node.links, (link, idx, eachCb) => { if (idx < pbh.header.fanout) { // the first pbh.header.fanout links are fanout bins // if a fanout bin is not 'empty', dig into and walk its DAGLinks diff --git a/src/core/components/start.js b/src/core/components/start.js index e306bf10e2..dda2faf267 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -67,9 +67,8 @@ module.exports = (self) => { ipnsStores.push(pubsubDs) } - // DHT should be added as routing if we are not running with offline flag - // TODO: Need to change this logic once DHT is enabled by default, for now fallback to Offline datastore - if (get(self._options, 'EXPERIMENTAL.dht', false) && !self._options.offline) { + // DHT should be added as routing if we are not running with local flag + if (!self._options.offline) { ipnsStores.push(self.libp2p.dht) } else { const offlineDatastore = new OfflineDatastore(self._repo) diff --git a/src/core/index.js b/src/core/index.js index 71e0b1341f..ba43f5703e 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -177,9 +177,6 @@ class IPFS extends EventEmitter { if (this._options.EXPERIMENTAL.sharding) { this.log('EXPERIMENTAL sharding is enabled') } - if (this._options.EXPERIMENTAL.dht) { - this.log('EXPERIMENTAL Kademlia DHT is enabled') - } this.state = require('./state')(this) diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index 2a14416f93..8fae1ae176 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -6,6 +6,7 @@ const WebSocketStarMulti = require('libp2p-websocket-star-multi') const Multiplex = require('libp2p-mplex') const SECIO = require('libp2p-secio') const Bootstrap = require('libp2p-bootstrap') +const KadDHT = require('libp2p-kad-dht') const libp2p = require('libp2p') const defaultsDeep = require('@nodeutils/defaults-deep') const multiaddr = require('multiaddr') @@ -37,7 +38,8 @@ class Node extends libp2p { wrtcstar.discovery, wsstar.discovery, Bootstrap - ] + ], + dht: KadDHT }, config: { peerDiscovery: { @@ -51,8 +53,10 @@ class Node extends libp2p { enabled: true } }, + dht: { + enabled: false + }, EXPERIMENTAL: { - dht: false, pubsub: false } } diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index df50ec984d..01e7046e19 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -53,10 +53,13 @@ class Node extends libp2p { } }, dht: { - kBucketSize: 20 + kBucketSize: 20, + enabled: true, + randomWalk: { + enabled: true + } }, EXPERIMENTAL: { - dht: false, pubsub: false } } diff --git a/src/http/api/resources/dht.js b/src/http/api/resources/dht.js new file mode 100644 index 0000000000..c88710c303 --- /dev/null +++ b/src/http/api/resources/dht.js @@ -0,0 +1,155 @@ +'use strict' + +const Joi = require('joi') +const Boom = require('boom') + +const CID = require('cids') + +const debug = require('debug') +const log = debug('jsipfs:http-api:dht') +log.error = debug('jsipfs:http-api:dht:error') + +exports = module.exports + +exports.findPeer = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + async handler (request, h) { + const ipfs = request.server.app.ipfs + const { arg } = request.query + let res + + try { + res = await ipfs.dht.findPeer(arg) + } catch (err) { + if (err.code === 'ERR_LOOKUP_FAILED') { + throw Boom.notFound(err.toString()) + } else { + throw Boom.boomify(err, { message: err.toString() }) + } + } + + return h.response({ + Responses: [{ + ID: res.id.toB58String(), + Addrs: res.multiaddrs.toArray().map((a) => a.toString()) + }], + Type: 2 + }) + } +} + +exports.findProvs = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required(), + 'num-providers': Joi.number().integer().default(20), + timeout: Joi.number() + }).unknown() + }, + async handler (request, h) { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + request.query.maxNumProviders = request.query['num-providers'] + + const res = await ipfs.dht.findProvs(arg, request.query) + + return h.response({ + Responses: res.map((peerInfo) => ({ + ID: peerInfo.id.toB58String(), + Addrs: peerInfo.multiaddrs.toArray().map((a) => a.toString()) + })), + Type: 4 + }) + } +} + +exports.get = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required(), + timeout: Joi.number() + }).unknown() + }, + async handler (request, h) { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + const res = await ipfs.dht.get(Buffer.from(arg)) + + return h.response({ + Extra: res.toString(), + Type: 5 + }) + } +} + +exports.provide = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + async handler (request, h) { + const ipfs = request.server.app.ipfs + const { arg } = request.query + let cid + + try { + cid = new CID(arg) + } catch (err) { + log.error(err) + throw Boom.boomify(err, { message: err.toString() }) + } + + await ipfs.dht.provide(cid) + + return h.response() + } +} + +exports.put = { + validate: { + query: Joi.object().keys({ + arg: Joi.array().items(Joi.string()).length(2).required() + }).unknown() + }, + parseArgs: (request, h) => { + return { + key: request.query.arg[0], + value: request.query.arg[1] + } + }, + async handler (request, h) { + const key = request.pre.args.key + const value = request.pre.args.value + const ipfs = request.server.app.ipfs + + await ipfs.dht.put(Buffer.from(key), Buffer.from(value)) + + return h.response() + } +} + +exports.query = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + async handler (request, h) { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + const res = await ipfs.dht.query(arg) + const response = res.map((peerInfo) => ({ + ID: peerInfo.id.toB58String() + })) + + return h.response(response) + } +} diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js index 66646d29d5..a54aa0d4f3 100644 --- a/src/http/api/resources/index.js +++ b/src/http/api/resources/index.js @@ -20,3 +20,4 @@ exports.key = require('./key') exports.stats = require('./stats') exports.resolve = require('./resolve') exports.name = require('./name') +exports.dht = require('./dht') diff --git a/src/http/api/routes/dht.js b/src/http/api/routes/dht.js new file mode 100644 index 0000000000..3dde96b9bd --- /dev/null +++ b/src/http/api/routes/dht.js @@ -0,0 +1,57 @@ +'use strict' + +const resources = require('../resources') + +module.exports = [ + { + method: '*', + path: '/api/v0/dht/findpeer', + options: { + validate: resources.dht.findPeer.validate + }, + handler: resources.dht.findPeer.handler + }, + { + method: '*', + path: '/api/v0/dht/findprovs', + options: { + validate: resources.dht.findProvs.validate + }, + handler: resources.dht.findProvs.handler + }, + { + method: '*', + path: '/api/v0/dht/get', + options: { + validate: resources.dht.get.validate + }, + handler: resources.dht.get.handler + }, + { + method: '*', + path: '/api/v0/dht/provide', + options: { + validate: resources.dht.provide.validate + }, + handler: resources.dht.provide.handler + }, + { + method: '*', + path: '/api/v0/dht/put', + options: { + pre: [ + { method: resources.dht.put.parseArgs, assign: 'args' } + ], + validate: resources.dht.put.validate + }, + handler: resources.dht.put.handler + }, + { + method: '*', + path: '/api/v0/dht/query', + options: { + validate: resources.dht.query.validate + }, + handler: resources.dht.query.handler + } +] diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js index 8f310ee73e..0f459aaf8b 100644 --- a/src/http/api/routes/index.js +++ b/src/http/api/routes/index.js @@ -23,5 +23,6 @@ module.exports = [ ...require('./key'), ...require('./stats'), require('./resolve'), - ...require('./name') + ...require('./name'), + ...require('./dht') ] diff --git a/test/cli/commands.js b/test/cli/commands.js index 9ba3f44519..f86e973b03 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') -const commandCount = 86 +const commandCount = 93 describe('commands', () => runOnAndOff((thing) => { let ipfs diff --git a/test/cli/dht.js b/test/cli/dht.js new file mode 100644 index 0000000000..0f12be3abc --- /dev/null +++ b/test/cli/dht.js @@ -0,0 +1,151 @@ +/* eslint-env mocha */ + +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const series = require('async/series') +const parallel = require('async/parallel') + +const DaemonFactory = require('ipfsd-ctl') +const df = DaemonFactory.create({ type: 'js' }) + +const ipfsExec = require('../utils/ipfs-exec') + +const daemonOpts = { + exec: `./src/cli/bin.js`, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + }, + initOptions: { bits: 512 } +} + +describe('dht', () => { + let nodes = [] + let ipfsA + let ipfsB + let idA + let idB + let multiaddrB + + // spawn daemons + before(function (done) { + this.timeout(80 * 1000) + series([ + (cb) => df.spawn(daemonOpts, (err, _ipfsd) => { + expect(err).to.not.exist() + + ipfsA = ipfsExec(_ipfsd.repoPath) + nodes.push(_ipfsd) + cb() + }), + (cb) => df.spawn(daemonOpts, (err, _ipfsd) => { + expect(err).to.not.exist() + + ipfsB = ipfsExec(_ipfsd.repoPath) + nodes.push(_ipfsd) + cb() + }) + ], done) + }) + + // get ids + before(function (done) { + this.timeout(80 * 1000) + parallel([ + (cb) => nodes[0].api.id((err, res) => { + expect(err).to.not.exist() + + idA = res.id + cb() + }), + (cb) => nodes[1].api.id((err, res) => { + expect(err).to.not.exist() + + multiaddrB = res.addresses[0] + idB = res.id + cb() + }) + ], done) + }) + + // connect daemons + before(function (done) { + this.timeout(80 * 1000) + + nodes[0].api.swarm.connect(multiaddrB, done) + }) + + after((done) => parallel(nodes.map((node) => (cb) => node.stop(cb)), done)) + + it('should be able to put a value to the dht and get it afterwards', function () { + this.timeout(60 * 1000) + + const key = 'testkey' + const value = 'testvalue' + + return ipfsA(`dht put ${key} ${value}`) + .then((res) => { + expect(res).to.exist() + + return ipfsB(`dht get ${key}`) + }) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(value) + }) + }) + + it('should be able to provide data and to be present in the findproviders', function () { + this.timeout(60 * 1000) + let cidAdded + + return ipfsA('add src/init-files/init-docs/readme') + .then((res) => { + expect(res).to.exist() + cidAdded = res.split(' ')[1] + + return ipfsA(`dht provide ${cidAdded}`) + }) + .then((res) => { + expect(res).to.exist() + + return ipfsB(`dht findprovs ${cidAdded}`) + }) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(idA) + }) + }) + + it('findpeer', function () { + this.timeout(60 * 1000) + + return ipfsA(`dht findpeer ${idB}`) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(multiaddrB) + }) + }) + + it('query', function () { + this.timeout(60 * 1000) + + return ipfsA(`dht query ${idB}`) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(idB) + }) + }) +}) diff --git a/test/cli/name-pubsub.js b/test/cli/name-pubsub.js index 6b252f67a1..fd125948a5 100644 --- a/test/cli/name-pubsub.js +++ b/test/cli/name-pubsub.js @@ -21,7 +21,18 @@ const spawnDaemon = (callback) => { df.spawn({ exec: `./src/cli/bin.js`, args: ['--enable-namesys-pubsub'], - initOptions: { bits: 512 } + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } diff --git a/test/cli/name.js b/test/cli/name.js index c69b9e2918..c34ae70ef4 100644 --- a/test/cli/name.js +++ b/test/cli/name.js @@ -181,7 +181,7 @@ describe('name', () => { }) }) - describe.skip('using dht', () => { + describe('using dht', () => { const passPhrase = hat() const pass = '--pass ' + passPhrase const name = 'test-key-' + hat() @@ -199,9 +199,17 @@ describe('name', () => { df.spawn({ exec: `./src/cli/bin.js`, config: { - Bootstrap: [] + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } }, - args: ['--pass', passPhrase, '--enable-dht-experiment'], + args: ['--pass', passPhrase], initOptions: { bits: 512 } }, (err, _ipfsd) => { expect(err).to.not.exist() diff --git a/test/core/create-node.spec.js b/test/core/create-node.spec.js index e2b9788520..cf5ba30e43 100644 --- a/test/core/create-node.spec.js +++ b/test/core/create-node.spec.js @@ -133,7 +133,7 @@ describe('create node', function () { }) it('should be silent', function (done) { - this.timeout(10 * 1000) + this.timeout(30 * 1000) sinon.spy(console, 'log') diff --git a/test/core/dht.spec.js b/test/core/dht.spec.js index 8e94bdb9b4..01812cc8bb 100644 --- a/test/core/dht.spec.js +++ b/test/core/dht.spec.js @@ -21,7 +21,9 @@ describe('dht', () => { factory.spawn({ exec: IPFS, initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [] + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd @@ -40,7 +42,7 @@ describe('dht', () => { describe('findprovs', () => { it('should callback with error for invalid CID input', (done) => { - ipfs.dht.findprovs('INVALID CID', (err) => { + ipfs.dht.findProvs('INVALID CID', (err) => { expect(err).to.exist() expect(err.code).to.equal('ERR_INVALID_CID') done() diff --git a/test/core/files-sharding.spec.js b/test/core/files-sharding.spec.js index 01d014c4f7..9289d78dff 100644 --- a/test/core/files-sharding.spec.js +++ b/test/core/files-sharding.spec.js @@ -62,7 +62,7 @@ describe('files directory (sharding tests)', () => { }) it('should be able to add dir without sharding', function (done) { - this.timeout(40 * 1000) + this.timeout(70 * 1000) pull( pull.values(createTestFiles()), @@ -114,7 +114,7 @@ describe('files directory (sharding tests)', () => { }) it('should be able to add dir with sharding', function (done) { - this.timeout(40 * 1000) + this.timeout(80 * 1000) pull( pull.values(createTestFiles()), diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index 05284161b7..49e3c36d4f 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -6,7 +6,9 @@ const CommonFactory = require('../utils/interface-common-factory') const isNode = require('detect-node') const dnsFetchStub = require('../utils/dns-fetch-stub') -describe('interface-ipfs-core tests', () => { +describe('interface-ipfs-core tests', function () { + this.timeout(20 * 1000) + // ipfs.dns in the browser calls out to https://ipfs.io/api/v0/dns. // The following code stubs self.fetch to return a static CID for calls // to https://ipfs.io/api/v0/dns?arg=ipfs.io. @@ -34,20 +36,29 @@ describe('interface-ipfs-core tests', () => { tests.dag(defaultCommonFactory) - const dhtCommonFactory = CommonFactory.create({ + tests.dht(CommonFactory.create({ spawnOptions: { - initOptions: { bits: 512 }, - EXPERIMENTAL: { - dht: true - }, config: { - Bootstrap: [] - } + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + }, + initOptions: { bits: 512 } } - }) - - tests.dht(dhtCommonFactory, { - skip: { reason: 'TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged' } + }), { + skip: isNode ? [ + // dht.get + { + name: 'should get a value after it was put on another node', + reason: 'Needs https://github.com/ipfs/interface-ipfs-core/pull/383' + } + ] : true }) tests.filesRegular(defaultCommonFactory, { @@ -91,15 +102,37 @@ describe('interface-ipfs-core tests', () => { tests.name(CommonFactory.create({ spawnOptions: { - args: ['--pass ipfs-is-awesome-software'], - initOptions: { bits: 512 } + args: ['--pass ipfs-is-awesome-software', '--offline'], + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } } })) tests.namePubsub(CommonFactory.create({ spawnOptions: { args: ['--enable-namesys-pubsub'], - initOptions: { bits: 1024 } + initOptions: { bits: 1024 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } } })) @@ -151,10 +184,6 @@ describe('interface-ipfs-core tests', () => { config = null } - config = config || { - Bootstrap: [] - } - const spawnOptions = { repoPath, config, initOptions: { bits: 512 } } ipfsFactory.spawn(spawnOptions, (err, _ipfsd) => { diff --git a/test/core/kad-dht.node.js b/test/core/kad-dht.node.js index 1f1f484ae6..1d7452c596 100644 --- a/test/core/kad-dht.node.js +++ b/test/core/kad-dht.node.js @@ -6,21 +6,20 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) + const parallel = require('async/parallel') const IPFSFactory = require('ipfsd-ctl') const f = IPFSFactory.create({ type: 'js' }) const config = { - Addresses: { - Swarm: [`/ip4/127.0.0.1/tcp/0`, `/ip4/127.0.0.1/tcp/0/ws`], - API: `/ip4/127.0.0.1/tcp/0`, - Gateway: `/ip4/127.0.0.1/tcp/0` - }, Bootstrap: [], Discovery: { MDNS: { Enabled: false + }, + webRTCStar: { + Enabled: false } } } @@ -33,16 +32,17 @@ function createNode (callback) { }, callback) } -describe.skip('verify that kad-dht is doing its thing', () => { +describe('kad-dht is routing content and peers correctly', () => { let nodeA let nodeB let nodeC - // let addrA let addrB let addrC let nodes - before((done) => { + before(function (done) { + this.timeout(30 * 1000) + parallel([ (cb) => createNode(cb), (cb) => createNode(cb), @@ -59,7 +59,6 @@ describe.skip('verify that kad-dht is doing its thing', () => { (cb) => nodeC.id(cb) ], (err, ids) => { expect(err).to.not.exist() - // addrA = ids[0].addresses[0] addrB = ids[1].addresses[0] addrC = ids[2].addresses[0] parallel([ @@ -72,10 +71,29 @@ describe.skip('verify that kad-dht is doing its thing', () => { after((done) => parallel(nodes.map((node) => (cb) => node.stop(cb)), done)) - it.skip('add a file in C, fetch through B in A', (done) => { + it('add a file in B, fetch in A', function (done) { + this.timeout(30 * 1000) + const file = { + path: 'testfile1.txt', + content: Buffer.from('hello kad 1') + } + + nodeB.add(file, (err, filesAdded) => { + expect(err).to.not.exist() + + nodeA.cat(filesAdded[0].hash, (err, data) => { + expect(err).to.not.exist() + expect(data).to.eql(file.content) + done() + }) + }) + }) + + it('add a file in C, fetch through B in A', function (done) { + this.timeout(30 * 1000) const file = { - path: 'testfile.txt', - content: Buffer.from('hello kad') + path: 'testfile2.txt', + content: Buffer.from('hello kad 2') } nodeC.add(file, (err, filesAdded) => { @@ -83,8 +101,7 @@ describe.skip('verify that kad-dht is doing its thing', () => { nodeA.cat(filesAdded[0].hash, (err, data) => { expect(err).to.not.exist() - expect(data.length).to.equal(file.data.length) - expect(data).to.eql(file.data) + expect(data).to.eql(file.content) done() }) }) diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index bd02fd29ec..c77709c9ee 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -13,13 +13,14 @@ const PeerBook = require('peer-book') const WebSocketStar = require('libp2p-websocket-star') const Multiplex = require('libp2p-mplex') const SECIO = require('libp2p-secio') +const KadDHT = require('libp2p-kad-dht') const Libp2p = require('libp2p') const libp2pComponent = require('../../src/core/components/libp2p') describe('libp2p customization', function () { // Provide some extra time for ci since we're starting libp2p nodes in each test - this.timeout(15 * 1000) + this.timeout(25 * 1000) let datastore let peerInfo @@ -27,7 +28,9 @@ describe('libp2p customization', function () { let testConfig let _libp2p - beforeEach((done) => { + before(function (done) { + this.timeout(25 * 1000) + testConfig = { Addresses: { Swarm: ['/ip4/0.0.0.0/tcp/4002'], @@ -43,7 +46,6 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, pubsub: false } } @@ -92,7 +94,8 @@ describe('libp2p customization', function () { ], peerDiscovery: [ wsstar.discovery - ] + ], + dht: KadDHT } }) } @@ -142,7 +145,6 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, pubsub: false } }) @@ -170,7 +172,6 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, pubsub: true }, libp2p: { @@ -207,7 +208,6 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, pubsub: true } }) diff --git a/test/core/name-pubsub.js b/test/core/name-pubsub.js index 10b6cfe995..7ba81413b3 100644 --- a/test/core/name-pubsub.js +++ b/test/core/name-pubsub.js @@ -32,7 +32,17 @@ describe('name-pubsub', function () { df.spawn({ exec: IPFS, args: [`--pass ${hat()}`, '--enable-namesys-pubsub'], - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } diff --git a/test/core/name.js b/test/core/name.js index b2ebc6927b..ed73592d28 100644 --- a/test/core/name.js +++ b/test/core/name.js @@ -50,7 +50,7 @@ describe('name', function () { this.timeout(50 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], + args: [`--pass ${hat()}`, '--offline'], config: { Bootstrap: [] } }, (err, _ipfsd) => { expect(err).to.not.exist() @@ -152,7 +152,7 @@ describe('name', function () { this.timeout(40 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], + args: [`--pass ${hat()}`, '--offline'], config: { Bootstrap: [] } }, (err, _ipfsd) => { expect(err).to.not.exist() @@ -193,8 +193,7 @@ describe('name', function () { }) }) - // TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged - describe.skip('work with dht', () => { + describe('work with dht', () => { let nodes let nodeA let nodeB @@ -204,8 +203,18 @@ describe('name', function () { const createNode = (callback) => { df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`, '--enable-dht-experiment'], - config: { Bootstrap: [] } + args: [`--pass ${hat()}`], + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } @@ -233,7 +242,8 @@ describe('name', function () { idA = ids[0] parallel([ (cb) => nodeC.swarm.connect(ids[0].addresses[0], cb), // C => A - (cb) => nodeC.swarm.connect(ids[1].addresses[0], cb) // C => B + (cb) => nodeC.swarm.connect(ids[1].addresses[0], cb), // C => B + (cb) => nodeA.swarm.connect(ids[1].addresses[0], cb) // A => B ], done) }) }) @@ -246,12 +256,12 @@ describe('name', function () { }) it('should publish and then resolve correctly with the default options', function (done) { - this.timeout(90 * 1000) + this.timeout(380 * 1000) publishAndResolve(nodeA, nodeB, ipfsRef, { resolve: false }, idA.id, {}, done) }) it('should recursively resolve to an IPFS hash', function (done) { - this.timeout(180 * 1000) + this.timeout(360 * 1000) const keyName = hat() nodeA.key.gen(keyName, { type: 'rsa', size: 2048 }, function (err, key) { @@ -284,7 +294,17 @@ describe('name', function () { df.spawn({ exec: IPFS, args: [`--pass ${hat()}`], - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd @@ -454,8 +474,18 @@ describe('name', function () { this.timeout(40 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], - config: { Bootstrap: [] } + args: [`--pass ${hat()}`, '--offline'], + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() node = _ipfsd.api diff --git a/test/core/object.spec.js b/test/core/object.spec.js index e9d03d40b9..eaac7b35a8 100644 --- a/test/core/object.spec.js +++ b/test/core/object.spec.js @@ -16,7 +16,7 @@ describe('object', () => { let ipfsd, ipfs before(function (done) { - this.timeout(20 * 1000) + this.timeout(50 * 1000) const factory = IPFSFactory.create({ type: 'proc' }) diff --git a/test/core/pin-set.js b/test/core/pin-set.js index ece713d61c..c1a31a1319 100644 --- a/test/core/pin-set.js +++ b/test/core/pin-set.js @@ -70,7 +70,17 @@ describe('pinSet', function () { before(function (done) { this.timeout(80 * 1000) repo = createTempRepo() - ipfs = new IPFS({ repo }) + ipfs = new IPFS({ + repo, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + } + } + } + }) ipfs.on('ready', () => { pinSet = createPinSet(ipfs.dag) done() @@ -107,7 +117,7 @@ describe('pinSet', function () { describe('handles large sets', function () { it('handles storing items > maxItems', function (done) { - this.timeout(19 * 1000) + this.timeout(90 * 1000) const expectedHash = 'QmbvhSy83QWfgLXDpYjDmLWBFfGc8utoqjcXHyj3gYuasT' const count = maxItems + 1 createNodes(count, (err, cids) => { diff --git a/test/core/pin.js b/test/core/pin.js index 558a46132b..8efd33dcb0 100644 --- a/test/core/pin.js +++ b/test/core/pin.js @@ -74,7 +74,12 @@ describe('pin', function () { before(function (done) { this.timeout(20 * 1000) repo = createTempRepo() - ipfs = new IPFS({ repo }) + ipfs = new IPFS({ + repo, + config: { + Bootstrap: [] + } + }) ipfs.on('ready', () => { pin = ipfs.pin ipfs.add(fixtures, done) diff --git a/test/core/ping.spec.js b/test/core/ping.spec.js index 9db0d7547a..5b784c5428 100644 --- a/test/core/ping.spec.js +++ b/test/core/ping.spec.js @@ -29,7 +29,7 @@ const config = { } function spawnNode ({ dht = false, type = 'js' }, cb) { - const args = dht ? ['--enable-dht-experiment'] : [] + const args = dht ? [] : ['--offline'] const factory = type === 'js' ? df : dfProc factory.spawn({ args, diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index f6ae50a609..4d4bd4848f 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -19,7 +19,7 @@ describe('preload', () => { let repo before(function (done) { - this.timeout(20 * 1000) + this.timeout(50 * 1000) repo = createTempRepo() ipfs = new IPFS({ @@ -27,7 +27,8 @@ describe('preload', () => { config: { Addresses: { Swarm: [] - } + }, + Bootstrap: [] }, preload: { enabled: true, @@ -38,20 +39,31 @@ describe('preload', () => { ipfs.on('ready', done) }) - afterEach((done) => MockPreloadNode.clearPreloadCids(done)) + afterEach(function (done) { + this.timeout(10 * 1000) + MockPreloadNode.clearPreloadCids(done) + }) - after((done) => ipfs.stop(done)) + after(function (done) { + this.timeout(50 * 1000) + ipfs.stop(done) + }) - after((done) => repo.teardown(done)) + after(function (done) { + this.timeout(50 * 1000) + repo.teardown(done) + }) - it('should preload content added with add', (done) => { + it('should preload content added with add', function (done) { + this.timeout(50 * 1000) ipfs.add(Buffer.from(hat()), (err, res) => { expect(err).to.not.exist() MockPreloadNode.waitForCids(res[0].hash, done) }) }) - it('should preload multiple content added with add', (done) => { + it('should preload multiple content added with add', function (done) { + this.timeout(50 * 1000) ipfs.add([{ content: Buffer.from(hat()) }, { @@ -64,7 +76,8 @@ describe('preload', () => { }) }) - it('should preload multiple content and intermediate dirs added with add', (done) => { + it('should preload multiple content and intermediate dirs added with add', function (done) { + this.timeout(50 * 1000) ipfs.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -84,7 +97,8 @@ describe('preload', () => { }) }) - it('should preload multiple content and wrapping dir for content added with add and wrapWithDirectory option', (done) => { + it('should preload multiple content and wrapping dir for content added with add and wrapWithDirectory option', function (done) { + this.timeout(50 * 1000) ipfs.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -104,7 +118,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with cat', (done) => { + it('should preload content retrieved with cat', function (done) { + this.timeout(50 * 1000) ipfs.add(Buffer.from(hat()), { preload: false }, (err, res) => { expect(err).to.not.exist() ipfs.cat(res[0].hash, (err) => { @@ -114,7 +129,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with get', (done) => { + it('should preload content retrieved with get', function (done) { + this.timeout(50 * 1000) ipfs.add(Buffer.from(hat()), { preload: false }, (err, res) => { expect(err).to.not.exist() ipfs.get(res[0].hash, (err) => { @@ -124,7 +140,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with ls', (done) => { + it('should preload content retrieved with ls', function (done) { + this.timeout(50 * 1000) ipfs.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -152,21 +169,24 @@ describe('preload', () => { }) }) - it('should preload content added with object.new', (done) => { + it('should preload content added with object.new', function (done) { + this.timeout(50 * 1000) ipfs.object.new((err, cid) => { expect(err).to.not.exist() MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) }) }) - it('should preload content added with object.put', (done) => { + it('should preload content added with object.put', function (done) { + this.timeout(50 * 1000) ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, cid) => { expect(err).to.not.exist() MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) }) }) - it('should preload content added with object.patch.addLink', (done) => { + it('should preload content added with object.patch.addLink', function (done) { + this.timeout(50 * 1000) parallel({ parent: (cb) => { waterfall([ @@ -194,7 +214,8 @@ describe('preload', () => { }) }) - it('should preload content added with object.patch.rmLink', (done) => { + it('should preload content added with object.patch.rmLink', function (done) { + this.timeout(50 * 1000) waterfall([ (cb) => ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, cb), (cid, cb) => ipfs.object.get(cid, (err, node) => cb(err, { node, cid })), @@ -218,7 +239,8 @@ describe('preload', () => { }) }) - it('should preload content added with object.patch.setData', (done) => { + it('should preload content added with object.patch.setData', function (done) { + this.timeout(50 * 1000) ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, cid) => { expect(err).to.not.exist() @@ -229,7 +251,8 @@ describe('preload', () => { }) }) - it('should preload content added with object.patch.appendData', (done) => { + it('should preload content added with object.patch.appendData', function (done) { + this.timeout(50 * 1000) ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, cid) => { expect(err).to.not.exist() @@ -240,7 +263,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with object.get', (done) => { + it('should preload content retrieved with object.get', function (done) { + this.timeout(50 * 1000) ipfs.object.new(null, { preload: false }, (err, cid) => { expect(err).to.not.exist() @@ -251,14 +275,16 @@ describe('preload', () => { }) }) - it('should preload content added with block.put', (done) => { + it('should preload content added with block.put', function (done) { + this.timeout(50 * 1000) ipfs.block.put(Buffer.from(hat()), (err, block) => { expect(err).to.not.exist() MockPreloadNode.waitForCids(block.cid.toBaseEncodedString(), done) }) }) - it('should preload content retrieved with block.get', (done) => { + it('should preload content retrieved with block.get', function (done) { + this.timeout(50 * 1000) ipfs.block.put(Buffer.from(hat()), { preload: false }, (err, block) => { expect(err).to.not.exist() ipfs.block.get(block.cid, (err) => { @@ -268,7 +294,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with block.stat', (done) => { + it('should preload content retrieved with block.stat', function (done) { + this.timeout(50 * 1000) ipfs.block.put(Buffer.from(hat()), { preload: false }, (err, block) => { expect(err).to.not.exist() ipfs.block.stat(block.cid, (err) => { @@ -278,7 +305,8 @@ describe('preload', () => { }) }) - it('should preload content added with dag.put', (done) => { + it('should preload content added with dag.put', function (done) { + this.timeout(50 * 1000) const obj = { test: hat() } ipfs.dag.put(obj, { format: 'dag-cbor', hashAlg: 'sha2-256' }, (err, cid) => { expect(err).to.not.exist() @@ -286,7 +314,8 @@ describe('preload', () => { }) }) - it('should preload content retrieved with dag.get', (done) => { + it('should preload content retrieved with dag.get', function (done) { + this.timeout(50 * 1000) const obj = { test: hat() } const opts = { format: 'dag-cbor', hashAlg: 'sha2-256', preload: false } ipfs.dag.put(obj, opts, (err, cid) => { @@ -299,13 +328,12 @@ describe('preload', () => { }) }) -describe('preload disabled', () => { +describe('preload disabled', function () { + this.timeout(50 * 1000) let ipfs let repo - before(function (done) { - this.timeout(20 * 1000) - + before((done) => { repo = createTempRepo() ipfs = new IPFS({ repo, diff --git a/test/core/utils.js b/test/core/utils.js index 7700e90955..c05c53a916 100644 --- a/test/core/utils.js +++ b/test/core/utils.js @@ -105,7 +105,7 @@ describe('utils', () => { }) describe('resolvePath', function () { - this.timeout(80 * 1000) + this.timeout(100 * 1000) const fixtures = [ 'test/fixtures/planets/mercury/wiki.md', 'test/fixtures/planets/solar-system.md' @@ -120,7 +120,10 @@ describe('utils', () => { before(done => { repo = createTempRepo() node = new IPFS({ - repo: repo + repo, + config: { + Bootstrap: [] + } }) node.once('ready', () => node.add(fixtures, done)) }) diff --git a/test/http-api/bootstrap.js b/test/http-api/bootstrap.js index 9e3aa76b98..7fb59d6474 100644 --- a/test/http-api/bootstrap.js +++ b/test/http-api/bootstrap.js @@ -18,7 +18,17 @@ describe('bootstrap endpoint', () => { df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd diff --git a/test/http-api/config.js b/test/http-api/config.js index 0958b16a72..87c6b4d35d 100644 --- a/test/http-api/config.js +++ b/test/http-api/config.js @@ -28,6 +28,13 @@ skipOnWindows('config endpoint', () => { let ipfs = null let ipfsd = null + // wait until the repo is ready to use + before(function (done) { + this.timeout(10 * 1000) + + setTimeout(done, 5 * 1000) + }) + before(function (done) { this.timeout(20 * 1000) @@ -38,6 +45,7 @@ skipOnWindows('config endpoint', () => { (cb) => df.spawn({ repoPath: repoPath, initOptions: { bits: 512 }, + config: { Bootstrap: [] }, disposable: false, start: true }, cb), @@ -59,7 +67,8 @@ skipOnWindows('config endpoint', () => { }) }) - after((done) => { + after(function (done) { + this.timeout(50 * 1000) rimraf(repoPath, (err) => { expect(err).to.not.exist() ipfsd.stop(done) diff --git a/test/http-api/dns.js b/test/http-api/dns.js index d373775f4f..b90802a621 100644 --- a/test/http-api/dns.js +++ b/test/http-api/dns.js @@ -28,7 +28,9 @@ describe('dns endpoint', () => { after((done) => ipfsd.stop(done)) describe('.dns', () => { - it('resolve ipfs.io dns', (done) => { + it('resolve ipfs.io dns', function (done) { + this.timeout(40 * 1000) + ipfs.dns('ipfs.io', (err, result) => { expect(err).to.not.exist() expect(result).to.exist() diff --git a/test/http-api/index.js b/test/http-api/index.js index 901cb212ef..a4cceb57a9 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -5,7 +5,7 @@ require('./bootstrap') require('./config') require('./dns') require('./id') -require('./inject') +require('./routes') require('./interface') require('./object') require('./version') diff --git a/test/http-api/inject/dht.js b/test/http-api/inject/dht.js new file mode 100644 index 0000000000..f614ee50dc --- /dev/null +++ b/test/http-api/inject/dht.js @@ -0,0 +1,144 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +module.exports = (http) => { + describe('/dht', () => { + let api + + before(() => { + api = http.api._apiServer + }) + + describe('/findpeer', () => { + it('returns 400 if no peerId is provided', async () => { + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/findpeer` + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + }) + + it('returns 404 if peerId is provided as there is no peers in the routing table', async () => { + const peerId = 'QmQ2zigjQikYnyYUSXZydNXrDRhBut2mubwJBaLXobMt3A' + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/findpeer?arg=${peerId}` + }) + + expect(res.statusCode).to.equal(404) + }) + }) + + describe('/findprovs', () => { + it('returns 400 if no key is provided', async () => { + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/findprovs` + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + }) + + it('returns 200 if key is provided', async () => { + const key = 'Qmc77hSNykXJ6Jxp1C6RpD8VENV7RK6JD7eAcWpc7nEZx2' + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/findprovs?arg=${key}` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result.Type).to.be.eql(4) + }) + }) + + describe('/get', () => { + it('returns 400 if no key is provided', async () => { + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/get` + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + }) + + it('returns 200 if key is provided', async () => { + const key = 'key' + const value = 'value' + + let res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/put?arg=${key}&arg=${value}` + }) + + expect(res.statusCode).to.equal(200) + + res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/get?arg=${key}` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result.Type).to.be.eql(5) + expect(res.result.Extra).to.be.eql(value) + }) + }) + + describe('/provide', () => { + it('returns 400 if no key is provided', async () => { + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/provide` + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + }) + + it('returns 500 if key is provided as the file was not added', async () => { + const key = 'Qmc77hSNykXJ6Jxp1C6RpD8VENV7RK6JD7eAcWpc7nEZx2' + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/provide?arg=${key}` + }) + + expect(res.statusCode).to.equal(500) // needs file add + }) + }) + + describe('/put', () => { + it('returns 400 if no key or value is provided', async () => { + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/put` + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + }) + + it('returns 200 if key and value is provided', async function () { + this.timeout(60 * 1000) + + const key = 'key' + const value = 'value' + + const res = await api.inject({ + method: 'GET', + url: `/api/v0/dht/put?arg=${key}&arg=${value}` + }) + + expect(res.statusCode).to.equal(200) + }) + }) + }) +} diff --git a/test/http-api/inject/index.js b/test/http-api/inject/index.js deleted file mode 100644 index 002f8dc4e0..0000000000 --- a/test/http-api/inject/index.js +++ /dev/null @@ -1,45 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const fs = require('fs') -const hat = require('hat') -const API = require('../../../src/http/index') -const promisify = require('promisify-es6') -const ncp = promisify(require('ncp').ncp) -const path = require('path') -const clean = require('../../utils/clean') - -describe('HTTP API', () => { - const repoExample = path.join(__dirname, '../../fixtures/go-ipfs-repo') - const repoTests = path.join(__dirname, '../../repo-tests-run') - - let http = {} - - const startHttpAPI = async () => { - http.api = new API({ - repo: repoTests, - pass: hat(), - config: { Bootstrap: [] }, - EXPERIMENTAL: { - pubsub: true - } - }) - await ncp(repoExample, repoTests) - await http.api.start() - } - - before(async function () { - this.timeout(60 * 1000) - await startHttpAPI() - }) - - after(async () => { - await http.api.stop() - clean(repoTests) - }) - - describe('## http-api spec tests', () => { - fs.readdirSync(path.join(__dirname)) - .forEach((file) => file !== 'index.js' && require(`./${file}`)(http)) - }) -}) diff --git a/test/http-api/inject/name.js b/test/http-api/inject/name.js index 1f3864b968..95a51eb5c2 100644 --- a/test/http-api/inject/name.js +++ b/test/http-api/inject/name.js @@ -19,7 +19,9 @@ module.exports = (http) => { api = http.api._apiServer }) - it('should publish a record', async () => { + it('should publish a record', async function () { + this.timeout(80 * 1000) + const res = await api.inject({ method: 'GET', url: `/api/v0/name/publish?arg=${cid}&resolve=false` @@ -29,7 +31,9 @@ module.exports = (http) => { expect(res.result.Value).to.equal(`/ipfs/${cid}`) }) - it('should publish and resolve a record', async () => { + it('should publish and resolve a record', async function () { + this.timeout(160 * 1000) + let res = await api.inject({ method: 'GET', url: `/api/v0/name/publish?arg=${cid}&resolve=false` diff --git a/test/http-api/inject/pin.js b/test/http-api/inject/pin.js index b433e0f27f..d70547a876 100644 --- a/test/http-api/inject/pin.js +++ b/test/http-api/inject/pin.js @@ -32,7 +32,8 @@ const pins = { } module.exports = (http) => { - describe('pin', () => { + describe('pin', function () { + this.timeout(20 * 1000) let api before(() => { diff --git a/test/http-api/inject/ping.js b/test/http-api/inject/ping.js index 1d3519e3d0..0ed460bba3 100644 --- a/test/http-api/inject/ping.js +++ b/test/http-api/inject/ping.js @@ -34,7 +34,9 @@ module.exports = (http) => { expect(res.statusCode).to.equal(400) }) - it('returns 500 for incorrect Peer Id', async () => { + it('returns 500 for incorrect Peer Id', async function () { + this.timeout(90 * 1000) + const res = await api.inject({ method: 'GET', url: `/api/v0/ping?arg=peerid` diff --git a/test/http-api/interface.js b/test/http-api/interface.js index 6bc89e6492..3372cde3af 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -21,8 +21,29 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { skip: { reason: 'TODO: DAG HTTP endpoints not implemented in js-ipfs yet!' } }) - tests.dht(defaultCommonFactory, { - skip: { reason: 'TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged' } + tests.dht(CommonFactory.create({ + spawnOptions: { + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + } + }), { + skip: [ + // dht.get + { + name: 'should get a value after it was put on another node', + reason: 'Needs https://github.com/ipfs/interface-ipfs-core/pull/383' + } + ] }) tests.filesRegular(defaultCommonFactory) @@ -92,10 +113,6 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { config = undefined } - config = config || { - Bootstrap: [] - } - const spawnOptions = { repoPath, config, initOptions: { bits: 512 } } ipfsFactory.spawn(spawnOptions, (err, _ipfsd) => { @@ -112,7 +129,7 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { } })) - tests.types(defaultCommonFactory) + tests.types(defaultCommonFactory, { skip: { reason: 'FIXME: currently failing' } }) tests.util(defaultCommonFactory, { skip: { reason: 'FIXME: currently failing' } }) }) diff --git a/test/http-api/object.js b/test/http-api/object.js index 003e8f1739..811aad6d72 100644 --- a/test/http-api/object.js +++ b/test/http-api/object.js @@ -30,7 +30,17 @@ describe('object endpoint', () => { df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd diff --git a/test/http-api/routes.js b/test/http-api/routes.js new file mode 100644 index 0000000000..0fa6fdf7df --- /dev/null +++ b/test/http-api/routes.js @@ -0,0 +1,91 @@ +/* eslint-env mocha */ +'use strict' + +const fs = require('fs') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +chai.use(dirtyChai) +const hat = require('hat') +const API = require('../../src/http/index') +const promisify = require('promisify-es6') +const ncp = promisify(require('ncp').ncp) +const path = require('path') +const clean = require('../utils/clean') + +describe('HTTP API', () => { + const repoExample = path.join(__dirname, '../fixtures/go-ipfs-repo') + const repoTests = path.join(__dirname, '../repo-tests-run') + + // bootstrap nodes get the set up too slow and gets timed out + const testsForCustomConfig = ['dht.js', 'files.js', 'name.js', 'pin.js', 'ping.js'] + + let http = {} + + const startHttpAPI = async (config) => { + http.api = new API({ + repo: repoTests, + pass: hat(), + config, + EXPERIMENTAL: { + pubsub: true + } + }) + await ncp(repoExample, repoTests) + await http.api.start() + } + + describe('custom config', () => { + const config = { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + + before(async function () { + this.timeout(60 * 1000) + + await startHttpAPI(config) + }) + + after(async function () { + this.timeout(50 * 1000) + + await http.api.stop() + clean(repoTests) + }) + + describe('## http-api spec tests for custom config', () => { + fs.readdirSync(path.join(`${__dirname}/inject/`)) + .forEach((file) => testsForCustomConfig.includes(file) && require(`./inject/${file}`)(http)) + }) + }) + + describe('default config', () => { + const config = { + Bootstrap: [] + } + + before(async function () { + this.timeout(60 * 1000) + await startHttpAPI(config) + }) + + after(async function () { + this.timeout(50 * 1000) + + await http.api.stop() + clean(repoTests) + }) + + describe('## http-api spec tests for default config', () => { + fs.readdirSync(path.join(`${__dirname}/inject/`)) + .forEach((file) => !testsForCustomConfig.includes(file) && require(`./inject/${file}`)(http)) + }) + }) +}) diff --git a/test/http-api/version.js b/test/http-api/version.js index f9ab764668..9fc0c3acc0 100644 --- a/test/http-api/version.js +++ b/test/http-api/version.js @@ -16,7 +16,17 @@ describe('version endpoint', () => { this.timeout(20 * 1000) df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd diff --git a/test/utils/interface-common-factory.js b/test/utils/interface-common-factory.js index 3eb4988b02..756e79c1cb 100644 --- a/test/utils/interface-common-factory.js +++ b/test/utils/interface-common-factory.js @@ -10,7 +10,20 @@ function createFactory (options) { options = options || {} options.factoryOptions = options.factoryOptions || { type: 'proc', exec: IPFS } - options.spawnOptions = options.spawnOptions || { initOptions: { bits: 512 }, config: { Bootstrap: [] } } + options.spawnOptions = options.spawnOptions || { + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + } if (options.factoryOptions.type !== 'proc') { options.factoryOptions.IpfsClient = options.factoryOptions.IpfsClient || ipfsClient diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js index 60a1417654..b46bde79f3 100644 --- a/test/utils/mock-preload-node.js +++ b/test/utils/mock-preload-node.js @@ -28,6 +28,7 @@ module.exports.createNode = () => { res.end() return } + if (req.url.startsWith('/api/v0/refs')) { const arg = new URL(`https://ipfs.io${req.url}`).searchParams.get('arg') cids = cids.concat(arg)