From 73452400c0f5712b6a133d880932405add8c4bdf Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Thu, 9 Nov 2017 19:24:00 +0000
Subject: [PATCH 01/10] big exporter overhaul: centralized dag reslving inside
 internal resolver stream

---
 package.json                     |  1 -
 src/exporter/dir-flat.js         | 19 +++------
 src/exporter/dir-hamt-sharded.js | 24 +++--------
 src/exporter/file.js             |  6 +--
 src/exporter/index.js            | 68 +++++++++++++++++++++++---------
 src/exporter/object.js           | 29 ++++++--------
 src/exporter/resolve.js          | 47 ++++++++++++++++++----
 src/importer/tree-builder.js     |  2 +-
 test/exporter-subtree.js         |  2 +-
 test/exporter.js                 |  2 +-
 10 files changed, 119 insertions(+), 81 deletions(-)

diff --git a/package.json b/package.json
index 50f540a8..fb2a8a0b 100644
--- a/package.json
+++ b/package.json
@@ -70,7 +70,6 @@
     "pull-batch": "^1.0.0",
     "pull-block": "1.2.0",
     "pull-cat": "^1.1.11",
-    "pull-defer": "~0.2.2",
     "pull-pair": "^1.1.0",
     "pull-paramap": "^1.2.2",
     "pull-pause": "0.0.1",
diff --git a/src/exporter/dir-flat.js b/src/exporter/dir-flat.js
index 9392018e..4ba23b85 100644
--- a/src/exporter/dir-flat.js
+++ b/src/exporter/dir-flat.js
@@ -1,14 +1,13 @@
 'use strict'
 
 const pull = require('pull-stream')
-const paramap = require('pull-paramap')
 const CID = require('cids')
 const cat = require('pull-cat')
 
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+function dirExporter (node, name, pathRest, resolve) {
   const accepts = pathRest[0]
 
   const dir = {
@@ -20,19 +19,13 @@ function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
     pull(
       pull.values(node.links),
       pull.map((link) => ({
-        linkName: link.name,
         path: name + '/' + link.name,
-        hash: link.multihash
+        multihash: link.multihash,
+        linkName: link.name,
+        pathRest: pathRest.slice(1)
       })),
       pull.filter((item) => accepts === undefined || item.linkName === accepts),
-      paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
-        if (err) {
-          return cb(err)
-        }
-
-        cb(null, resolve(n.value, accepts || item.path, pathRest, ipldResolver, name, parent))
-      })),
-      pull.flatten()
+      resolve
     )
   ]
 
@@ -41,7 +34,5 @@ function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
     streams.unshift(pull.values([dir]))
   }
 
-  pathRest.shift()
-
   return cat(streams)
 }
diff --git a/src/exporter/dir-hamt-sharded.js b/src/exporter/dir-hamt-sharded.js
index 1bf81c51..7039ff35 100644
--- a/src/exporter/dir-hamt-sharded.js
+++ b/src/exporter/dir-hamt-sharded.js
@@ -1,7 +1,6 @@
 'use strict'
 
 const pull = require('pull-stream')
-const paramap = require('pull-paramap')
 const CID = require('cids')
 const cat = require('pull-cat')
 const cleanHash = require('./clean-multihash')
@@ -9,7 +8,7 @@ const cleanHash = require('./clean-multihash')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+function shardedDirExporter (node, name, pathRest, resolve, dag, parent) {
   let dir
   if (!parent || parent.path !== name) {
     dir = [{
@@ -37,29 +36,16 @@ function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent
             fromPathRest: fromPathRest,
             name: p,
             path: pp,
-            hash: link.multihash,
-            pathRest: p ? pathRest.slice(1) : pathRest
+            multihash: link.multihash,
+            pathRest: p ? pathRest.slice(1) : pathRest,
+            parent: (dir && dir[0]) || parent
           }
         } else {
           return ''
         }
       }),
       pull.filter(Boolean),
-      paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
-        if (err) {
-          return cb(err)
-        }
-
-        cb(
-          null,
-          resolve(
-            n.value,
-            item.fromPathRest ? item.name : item.path,
-            item.pathRest,
-            ipldResolver,
-            (dir && dir[0]) || parent))
-      })),
-      pull.flatten()
+      resolve
     )
   ]
 
diff --git a/src/exporter/file.js b/src/exporter/file.js
index 67763073..868c407d 100644
--- a/src/exporter/file.js
+++ b/src/exporter/file.js
@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, pathRest, ipldResolver) => {
+module.exports = (node, name, pathRest, resolve, dag) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -20,12 +20,12 @@ module.exports = (node, name, pathRest, ipldResolver) => {
   function visitor (node) {
     return pull(
       pull.values(node.links),
-      paramap((link, cb) => ipldResolver.get(new CID(link.multihash), cb)),
+      paramap((link, cb) => dag.get(new CID(link.multihash), cb)),
       pull.map((result) => result.value)
     )
   }
 
-  const accepts = pathRest.shift()
+  const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== name) {
     return pull.empty()
diff --git a/src/exporter/index.js b/src/exporter/index.js
index 08017c9f..5933b211 100644
--- a/src/exporter/index.js
+++ b/src/exporter/index.js
@@ -2,9 +2,9 @@
 
 const pull = require('pull-stream')
 const CID = require('cids')
-const pullDefer = require('pull-defer')
+const b58Encode = require('bs58').encode
 
-const resolve = require('./resolve').resolve
+const createResolver = require('./resolve').createResolver
 
 function pathBaseAndRest (path) {
   // Buffer -> raw multihash or CID in buffer
@@ -36,28 +36,60 @@ function pathBaseAndRest (path) {
   }
 }
 
-module.exports = (path, dag) => {
+const defaultOptions = {
+  recurse: true
+}
+
+module.exports = (path, dag, _options) => {
+
+  const options = Object.assign({}, defaultOptions, _options)
+
+  let dPath
   try {
-    path = pathBaseAndRest(path)
+    dPath = pathBaseAndRest(path)
   } catch (err) {
     return pull.error(err)
   }
 
-  const d = pullDefer.source()
-
-  const cid = new CID(path.base)
+  return pull(
+    pull.values([{
+      multihash: new CID(dPath.base),
+      name: dPath.base,
+      path: dPath.base,
+      pathRest: dPath.rest
+    }]),
+    createResolver(dag, options),
+    pull.map((node) => ({
+      path: finalPathFor(node),
+      size: node.size,
+      hash: node.hash,
+      content: node.content
+    }))
+  )
 
-  dag.get(cid, (err, node) => {
-    if (err) {
-      return pull.error(err)
+  function finalPathFor(node) {
+    if (!dPath.rest.length) {
+      return node.path
     }
-    d.resolve(pull.values([node]))
-  })
 
-  return pull(
-    d,
-    pull.map((result) => result.value),
-    pull.map((node) => resolve(node, path.base, path.rest, dag)),
-    pull.flatten()
-  )
+    const cutElements = [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))
+    const lengthToCut = join(cutElements).length
+    let retPath = node.path.substring(lengthToCut)
+    if (retPath.charAt(0) === '/') {
+      retPath = retPath.substring(1)
+    }
+    if (!retPath) {
+      retPath = dPath.rest[dPath.rest.length - 1] || dPath.base
+    }
+    return retPath
+  }
 }
+
+function join(paths) {
+  return paths.reduce((acc, path) => {
+    if (acc.length) {
+      acc += '/'
+    }
+    return acc + path
+  }, '')
+}
\ No newline at end of file
diff --git a/src/exporter/object.js b/src/exporter/object.js
index af24a970..aa55ecd9 100644
--- a/src/exporter/object.js
+++ b/src/exporter/object.js
@@ -2,29 +2,26 @@
 
 const CID = require('cids')
 const pull = require('pull-stream')
-const pullDefer = require('pull-defer')
 
-module.exports = (node, name, pathRest, ipldResolver, resolve) => {
+module.exports = (node, name, pathRest, resolve, dag, parent) => {
   let newNode
   if (pathRest.length) {
-    const pathElem = pathRest.shift()
+    const pathElem = pathRest[0]
     newNode = node[pathElem]
     const newName = name + '/' + pathElem
-    if (CID.isCID(newNode)) {
-      const d = pullDefer.source()
-      ipldResolver.get(sanitizeCID(newNode), (err, newNode) => {
-        if (err) {
-          d.resolve(pull.error(err))
-        } else {
-          d.resolve(resolve(newNode.value, newName, pathRest, ipldResolver, node))
-        }
-      })
-      return d
-    } else if (newNode !== undefined) {
-      return resolve(newNode, newName, pathRest, ipldResolver, node)
-    } else {
+    if (!newNode) {
       return pull.error('not found')
     }
+    const isCID = CID.isCID(newNode)
+    return pull(
+      pull.values([{
+        path: newName,
+        pathRest: pathRest.slice(1),
+        multihash: isCID && newNode,
+        object: !isCID && newNode,
+        parent: parent
+      }]),
+      resolve)
   } else {
     return pull.error(new Error('invalid node type'))
   }
diff --git a/src/exporter/resolve.js b/src/exporter/resolve.js
index 71b1067a..121b3652 100644
--- a/src/exporter/resolve.js
+++ b/src/exporter/resolve.js
@@ -2,6 +2,8 @@
 
 const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
+const paramap = require('pull-paramap')
+const CID = require('cids')
 
 const resolvers = {
   directory: require('./dir-flat'),
@@ -11,17 +13,44 @@ const resolvers = {
 }
 
 module.exports = Object.assign({
-  resolve: resolve,
+  createResolver: createResolver,
   typeOf: typeOf
 }, resolvers)
 
-function resolve (node, hash, pathRest, ipldResolver, parentNode) {
-  const type = typeOf(node)
-  const resolver = resolvers[type]
-  if (!resolver) {
-    return pull.error(new Error('Unkown node type ' + type))
+function createResolver (dag, options, depth, parent) {
+  if (! depth) {
+    depth = 0
+  }
+
+  if (!options.recurse && depth > 0) {
+    return pull.map(identity)
+  }
+
+  return pull(
+    paramap((item, cb) => {
+      if (item.object) {
+        return cb(null, resolve(item.object, item.path, item.pathRest, dag, item.parent || parent))
+      }
+      dag.get(new CID(item.multihash), (err, node) => {
+        if (err) {
+          return cb(err)
+        }
+        const name = item.fromPathRest ? item.name : item.path
+        cb(null, resolve(node.value, name, item.pathRest, dag, item.parent || parent))
+      })
+    }),
+    pull.flatten()
+  )
+
+  function resolve (node, hash, pathRest, parentNode) {
+    const type = typeOf(node)
+    const nodeResolver = resolvers[type]
+    if (!nodeResolver) {
+      return pull.error(new Error('Unkown node type ' + type))
+    }
+    const resolveDeep = createResolver(dag, options, depth + 1, node)
+    return nodeResolver(node, hash, pathRest, resolveDeep, dag, parentNode)
   }
-  return resolver(node, hash, pathRest, ipldResolver, resolve, parentNode)
 }
 
 function typeOf (node) {
@@ -31,3 +60,7 @@ function typeOf (node) {
     return 'object'
   }
 }
+
+function identity (o) {
+  return o
+}
diff --git a/src/importer/tree-builder.js b/src/importer/tree-builder.js
index c04cb0ac..eaf5101f 100644
--- a/src/importer/tree-builder.js
+++ b/src/importer/tree-builder.js
@@ -91,7 +91,7 @@ function createTreeBuilder (ipldResolver, _options) {
   // ---- Add to tree
 
   function addToTree (elem, callback) {
-    const pathElems = elem.path.split('/').filter(notEmpty)
+    const pathElems = (elem.path || '').split('/').filter(notEmpty)
     let parent = tree
     const lastIndex = pathElems.length - 1
 
diff --git a/test/exporter-subtree.js b/test/exporter-subtree.js
index d2c59048..c6257d47 100644
--- a/test/exporter-subtree.js
+++ b/test/exporter-subtree.js
@@ -16,7 +16,7 @@ const exporter = unixFSEngine.exporter
 const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
 
 module.exports = (repo) => {
-  describe('exporter', function () {
+  describe('exporter subtree', () => {
     this.timeout(10 * 1000)
 
     let ipldResolver
diff --git a/test/exporter.js b/test/exporter.js
index eacbbb21..612a485f 100644
--- a/test/exporter.js
+++ b/test/exporter.js
@@ -117,8 +117,8 @@ module.exports = (repo) => {
       pull(
         exporter(hash, ipldResolver),
         pull.collect((err, files) => {
-          files.forEach(file => expect(file).to.have.property('hash'))
           expect(err).to.not.exist()
+          files.forEach(file => expect(file).to.have.property('hash'))
 
           expect(
             files.map((file) => file.path)

From b77240b23b80dc522bfc415b8960ca176d91432b Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 11:44:42 +0000
Subject: [PATCH 02/10] exporter: maxDepth instead of recursive flag

---
 src/exporter/dir-flat.js         |  9 +++++----
 src/exporter/dir-hamt-sharded.js |  9 ++++-----
 src/exporter/file.js             |  6 +++---
 src/exporter/index.js            | 23 ++++++++++++-----------
 src/exporter/object.js           |  4 ----
 src/exporter/resolve.js          |  8 ++++----
 test/exporter-subtree.js         |  2 +-
 7 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/src/exporter/dir-flat.js b/src/exporter/dir-flat.js
index 4ba23b85..a41b8f5a 100644
--- a/src/exporter/dir-flat.js
+++ b/src/exporter/dir-flat.js
@@ -1,17 +1,16 @@
 'use strict'
 
 const pull = require('pull-stream')
-const CID = require('cids')
 const cat = require('pull-cat')
 
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, pathRest, resolve) {
+function dirExporter (node, path, pathRest, resolve) {
   const accepts = pathRest[0]
 
   const dir = {
-    path: name,
+    path: path,
     hash: node.multihash
   }
 
@@ -19,7 +18,9 @@ function dirExporter (node, name, pathRest, resolve) {
     pull(
       pull.values(node.links),
       pull.map((link) => ({
-        path: name + '/' + link.name,
+        size: link.size,
+        name: link.name,
+        path: path + '/' + link.name,
         multihash: link.multihash,
         linkName: link.name,
         pathRest: pathRest.slice(1)
diff --git a/src/exporter/dir-hamt-sharded.js b/src/exporter/dir-hamt-sharded.js
index 7039ff35..b6521b70 100644
--- a/src/exporter/dir-hamt-sharded.js
+++ b/src/exporter/dir-hamt-sharded.js
@@ -1,18 +1,17 @@
 'use strict'
 
 const pull = require('pull-stream')
-const CID = require('cids')
 const cat = require('pull-cat')
 const cleanHash = require('./clean-multihash')
 
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, pathRest, resolve, dag, parent) {
+function shardedDirExporter (node, path, pathRest, resolve, dag, parent) {
   let dir
-  if (!parent || parent.path !== name) {
+  if (!parent || parent.path !== path) {
     dir = [{
-      path: name,
+      path: path,
       hash: cleanHash(node.multihash)
     }]
   }
@@ -23,7 +22,7 @@ function shardedDirExporter (node, name, pathRest, resolve, dag, parent) {
       pull.map((link) => {
         // remove the link prefix (2 chars for the bucket index)
         const p = link.name.substring(2)
-        const pp = p ? name + '/' + p : name
+        const pp = p ? path + '/' + p : path
         let accept = true
         let fromPathRest = false
 
diff --git a/src/exporter/file.js b/src/exporter/file.js
index 868c407d..5c1f8b3c 100644
--- a/src/exporter/file.js
+++ b/src/exporter/file.js
@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, pathRest, resolve, dag) => {
+module.exports = (node, path, pathRest, resolve, dag) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -27,7 +27,7 @@ module.exports = (node, name, pathRest, resolve, dag) => {
 
   const accepts = pathRest[0]
 
-  if (accepts !== undefined && accepts !== name) {
+  if (accepts !== undefined && accepts !== path) {
     return pull.empty()
   }
 
@@ -39,7 +39,7 @@ module.exports = (node, name, pathRest, resolve, dag) => {
   const file = UnixFS.unmarshal(node.data)
   return pull.values([{
     content: content,
-    path: name,
+    path: path,
     hash: node.multihash,
     size: file.fileSize()
   }])
diff --git a/src/exporter/index.js b/src/exporter/index.js
index 5933b211..d3775031 100644
--- a/src/exporter/index.js
+++ b/src/exporter/index.js
@@ -2,7 +2,6 @@
 
 const pull = require('pull-stream')
 const CID = require('cids')
-const b58Encode = require('bs58').encode
 
 const createResolver = require('./resolve').createResolver
 
@@ -37,11 +36,10 @@ function pathBaseAndRest (path) {
 }
 
 const defaultOptions = {
-  recurse: true
+  maxDepth: Infinity
 }
 
 module.exports = (path, dag, _options) => {
-
   const options = Object.assign({}, defaultOptions, _options)
 
   let dPath
@@ -51,6 +49,9 @@ module.exports = (path, dag, _options) => {
     return pull.error(err)
   }
 
+  const pathLengthToCut = join(
+    [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))).length
+
   return pull(
     pull.values([{
       multihash: new CID(dPath.base),
@@ -60,21 +61,21 @@ module.exports = (path, dag, _options) => {
     }]),
     createResolver(dag, options),
     pull.map((node) => ({
+      name: node.name,
       path: finalPathFor(node),
       size: node.size,
-      hash: node.hash,
-      content: node.content
+      hash: node.hash || node.multihash,
+      content: node.content,
+      type: node.type
     }))
   )
 
-  function finalPathFor(node) {
+  function finalPathFor (node) {
     if (!dPath.rest.length) {
       return node.path
     }
 
-    const cutElements = [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))
-    const lengthToCut = join(cutElements).length
-    let retPath = node.path.substring(lengthToCut)
+    let retPath = node.path.substring(pathLengthToCut)
     if (retPath.charAt(0) === '/') {
       retPath = retPath.substring(1)
     }
@@ -85,11 +86,11 @@ module.exports = (path, dag, _options) => {
   }
 }
 
-function join(paths) {
+function join (paths) {
   return paths.reduce((acc, path) => {
     if (acc.length) {
       acc += '/'
     }
     return acc + path
   }, '')
-}
\ No newline at end of file
+}
diff --git a/src/exporter/object.js b/src/exporter/object.js
index aa55ecd9..24dbebc9 100644
--- a/src/exporter/object.js
+++ b/src/exporter/object.js
@@ -26,7 +26,3 @@ module.exports = (node, name, pathRest, resolve, dag, parent) => {
     return pull.error(new Error('invalid node type'))
   }
 }
-
-function sanitizeCID (cid) {
-  return new CID(cid.version, cid.codec, cid.multihash)
-}
diff --git a/src/exporter/resolve.js b/src/exporter/resolve.js
index 121b3652..34e01932 100644
--- a/src/exporter/resolve.js
+++ b/src/exporter/resolve.js
@@ -18,11 +18,11 @@ module.exports = Object.assign({
 }, resolvers)
 
 function createResolver (dag, options, depth, parent) {
-  if (! depth) {
+  if (!depth) {
     depth = 0
   }
 
-  if (!options.recurse && depth > 0) {
+  if (depth > options.maxDepth) {
     return pull.map(identity)
   }
 
@@ -42,14 +42,14 @@ function createResolver (dag, options, depth, parent) {
     pull.flatten()
   )
 
-  function resolve (node, hash, pathRest, parentNode) {
+  function resolve (node, path, pathRest, parentNode) {
     const type = typeOf(node)
     const nodeResolver = resolvers[type]
     if (!nodeResolver) {
       return pull.error(new Error('Unkown node type ' + type))
     }
     const resolveDeep = createResolver(dag, options, depth + 1, node)
-    return nodeResolver(node, hash, pathRest, resolveDeep, dag, parentNode)
+    return nodeResolver(node, path, pathRest, resolveDeep, dag, parentNode)
   }
 }
 
diff --git a/test/exporter-subtree.js b/test/exporter-subtree.js
index c6257d47..70b92e07 100644
--- a/test/exporter-subtree.js
+++ b/test/exporter-subtree.js
@@ -17,7 +17,7 @@ const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
 
 module.exports = (repo) => {
   describe('exporter subtree', () => {
-    this.timeout(10 * 1000)
+    // this.timeout(10 * 1000)
 
     let ipldResolver
 

From 7de69fe9b0397be2c8cf99ea3e9e3c39aa04b3c0 Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 16:15:14 +0000
Subject: [PATCH 03/10] exporter: exposing name and depth

---
 src/exporter/dir-flat.js         | 11 ++++++++---
 src/exporter/dir-hamt-sharded.js | 21 +++++++++++----------
 src/exporter/file.js             |  7 +++++--
 src/exporter/index.js            | 23 ++++++++++++++---------
 src/exporter/object.js           |  6 ++++--
 src/exporter/resolve.js          | 23 ++++++++++++++++-------
 6 files changed, 58 insertions(+), 33 deletions(-)

diff --git a/src/exporter/dir-flat.js b/src/exporter/dir-flat.js
index a41b8f5a..b8a677de 100644
--- a/src/exporter/dir-flat.js
+++ b/src/exporter/dir-flat.js
@@ -6,24 +6,29 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, path, pathRest, resolve) {
+function dirExporter (node, name, path, pathRest, resolve, dag, parent, depth) {
   const accepts = pathRest[0]
 
   const dir = {
+    name: name,
+    depth: depth,
     path: path,
-    hash: node.multihash
+    hash: node.multihash,
+    type: 'dir'
   }
 
   const streams = [
     pull(
       pull.values(node.links),
       pull.map((link) => ({
+        depth: depth + 1,
         size: link.size,
         name: link.name,
         path: path + '/' + link.name,
         multihash: link.multihash,
         linkName: link.name,
-        pathRest: pathRest.slice(1)
+        pathRest: pathRest.slice(1),
+        type: 'dir'
       })),
       pull.filter((item) => accepts === undefined || item.linkName === accepts),
       resolve
diff --git a/src/exporter/dir-hamt-sharded.js b/src/exporter/dir-hamt-sharded.js
index b6521b70..22f20bf6 100644
--- a/src/exporter/dir-hamt-sharded.js
+++ b/src/exporter/dir-hamt-sharded.js
@@ -7,13 +7,16 @@ const cleanHash = require('./clean-multihash')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, path, pathRest, resolve, dag, parent) {
+function shardedDirExporter (node, name, path, pathRest, resolve, dag, parent, depth) {
   let dir
-  if (!parent || parent.path !== path) {
-    dir = [{
+  if (!parent || (parent.path !== path)) {
+    dir = {
+      name: name,
+      depth: depth,
       path: path,
-      hash: cleanHash(node.multihash)
-    }]
+      hash: cleanHash(node.multihash),
+      type: 'dir'
+    }
   }
 
   const streams = [
@@ -24,20 +27,18 @@ function shardedDirExporter (node, path, pathRest, resolve, dag, parent) {
         const p = link.name.substring(2)
         const pp = p ? path + '/' + p : path
         let accept = true
-        let fromPathRest = false
 
         if (p && pathRest.length) {
-          fromPathRest = true
           accept = (p === pathRest[0])
         }
         if (accept) {
           return {
-            fromPathRest: fromPathRest,
+            depth: depth + 1,
             name: p,
             path: pp,
             multihash: link.multihash,
             pathRest: p ? pathRest.slice(1) : pathRest,
-            parent: (dir && dir[0]) || parent
+            parent: dir || parent
           }
         } else {
           return ''
@@ -49,7 +50,7 @@ function shardedDirExporter (node, path, pathRest, resolve, dag, parent) {
   ]
 
   if (!pathRest.length) {
-    streams.unshift(pull.values(dir))
+    streams.unshift(pull.values([dir]))
   }
 
   return cat(streams)
diff --git a/src/exporter/file.js b/src/exporter/file.js
index 5c1f8b3c..134989d7 100644
--- a/src/exporter/file.js
+++ b/src/exporter/file.js
@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, path, pathRest, resolve, dag) => {
+module.exports = (node, name, path, pathRest, resolve, dag, parent, depth) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -38,9 +38,12 @@ module.exports = (node, path, pathRest, resolve, dag) => {
 
   const file = UnixFS.unmarshal(node.data)
   return pull.values([{
+    depth: depth,
     content: content,
+    name: name,
     path: path,
     hash: node.multihash,
-    size: file.fileSize()
+    size: file.fileSize(),
+    type: 'file'
   }])
 }
diff --git a/src/exporter/index.js b/src/exporter/index.js
index d3775031..14a174dd 100644
--- a/src/exporter/index.js
+++ b/src/exporter/index.js
@@ -57,17 +57,22 @@ module.exports = (path, dag, _options) => {
       multihash: new CID(dPath.base),
       name: dPath.base,
       path: dPath.base,
-      pathRest: dPath.rest
+      pathRest: dPath.rest,
+      depth: 0
     }]),
     createResolver(dag, options),
-    pull.map((node) => ({
-      name: node.name,
-      path: finalPathFor(node),
-      size: node.size,
-      hash: node.hash || node.multihash,
-      content: node.content,
-      type: node.type
-    }))
+    pull.filter(Boolean),
+    pull.map((node) => {
+      return {
+        depth: node.depth,
+        name: node.name,
+        path: finalPathFor(node),
+        size: node.size,
+        hash: node.hash || node.multihash,
+        content: node.content,
+        type: node.type
+      }
+    })
   )
 
   function finalPathFor (node) {
diff --git a/src/exporter/object.js b/src/exporter/object.js
index 24dbebc9..3f7f819f 100644
--- a/src/exporter/object.js
+++ b/src/exporter/object.js
@@ -3,18 +3,20 @@
 const CID = require('cids')
 const pull = require('pull-stream')
 
-module.exports = (node, name, pathRest, resolve, dag, parent) => {
+module.exports = (node, name, path, pathRest, resolve, dag, parent, depth) => {
   let newNode
   if (pathRest.length) {
     const pathElem = pathRest[0]
     newNode = node[pathElem]
-    const newName = name + '/' + pathElem
+    const newName = path + '/' + pathElem
     if (!newNode) {
       return pull.error('not found')
     }
     const isCID = CID.isCID(newNode)
     return pull(
       pull.values([{
+        depth: depth,
+        name: pathElem,
         path: newName,
         pathRest: pathRest.slice(1),
         multihash: isCID && newNode,
diff --git a/src/exporter/resolve.js b/src/exporter/resolve.js
index 34e01932..3ee01235 100644
--- a/src/exporter/resolve.js
+++ b/src/exporter/resolve.js
@@ -28,28 +28,37 @@ function createResolver (dag, options, depth, parent) {
 
   return pull(
     paramap((item, cb) => {
+      if ((typeof item.depth) !== 'number') {
+        return pull.error(new Error('no depth'))
+      }
       if (item.object) {
-        return cb(null, resolve(item.object, item.path, item.pathRest, dag, item.parent || parent))
+        return cb(null, resolveItem(item.object, item))
       }
       dag.get(new CID(item.multihash), (err, node) => {
         if (err) {
           return cb(err)
         }
-        const name = item.fromPathRest ? item.name : item.path
-        cb(null, resolve(node.value, name, item.pathRest, dag, item.parent || parent))
+        // const name = item.fromPathRest ? item.name : item.path
+        cb(null, resolveItem(node.value, item))
       })
     }),
-    pull.flatten()
+    pull.flatten(),
+    pull.filter(Boolean),
+    pull.filter((node) => node.depth <= options.maxDepth)
   )
 
-  function resolve (node, path, pathRest, parentNode) {
+  function resolveItem (node, item) {
+    return resolve(node, item.name, item.path, item.pathRest, dag, item.parent || parent, item.depth)
+  }
+
+  function resolve (node, name, path, pathRest, dag, parentNode, depth) {
     const type = typeOf(node)
     const nodeResolver = resolvers[type]
     if (!nodeResolver) {
       return pull.error(new Error('Unkown node type ' + type))
     }
-    const resolveDeep = createResolver(dag, options, depth + 1, node)
-    return nodeResolver(node, path, pathRest, resolveDeep, dag, parentNode)
+    const resolveDeep = createResolver(dag, options, depth, node)
+    return nodeResolver(node, name, path, pathRest, resolveDeep, dag, parentNode, depth)
   }
 }
 

From 4ad165c07df2cf936751def88bcb607c6292ce20 Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 16:21:04 +0000
Subject: [PATCH 04/10] exporter: exposing dir size

---
 src/exporter/dir-flat.js         | 1 +
 src/exporter/dir-hamt-sharded.js | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/exporter/dir-flat.js b/src/exporter/dir-flat.js
index b8a677de..46fa9323 100644
--- a/src/exporter/dir-flat.js
+++ b/src/exporter/dir-flat.js
@@ -14,6 +14,7 @@ function dirExporter (node, name, path, pathRest, resolve, dag, parent, depth) {
     depth: depth,
     path: path,
     hash: node.multihash,
+    size: node.size,
     type: 'dir'
   }
 
diff --git a/src/exporter/dir-hamt-sharded.js b/src/exporter/dir-hamt-sharded.js
index 22f20bf6..0f744756 100644
--- a/src/exporter/dir-hamt-sharded.js
+++ b/src/exporter/dir-hamt-sharded.js
@@ -15,6 +15,7 @@ function shardedDirExporter (node, name, path, pathRest, resolve, dag, parent, d
       depth: depth,
       path: path,
       hash: cleanHash(node.multihash),
+      size: node.size,
       type: 'dir'
     }
   }

From 381998c959b78c78d8772d35e640e46c9698737f Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 17:18:32 +0000
Subject: [PATCH 05/10] tests: increased timeout for importing big file

---
 test/importer.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/importer.js b/test/importer.js
index cdc74333..588d590b 100644
--- a/test/importer.js
+++ b/test/importer.js
@@ -311,7 +311,7 @@ module.exports = (repo) => {
             done()
           })
         )
-      })
+      }).timeout(60 * 1000)
 
       it('file bigger than a single chunk inside a dir', (done) => {
         pull(
@@ -331,7 +331,7 @@ module.exports = (repo) => {
             done()
           })
         )
-      })
+      }).timeout(60 * 1000)
 
       it('empty directory', (done) => {
         pull(

From c8552a498ec78fbd5ff848406f010f583276b98b Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 17:21:13 +0000
Subject: [PATCH 06/10] tests: test exporter maxDepth

---
 test/exporter.js | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/test/exporter.js b/test/exporter.js
index 612a485f..3558724e 100644
--- a/test/exporter.js
+++ b/test/exporter.js
@@ -151,6 +151,42 @@ module.exports = (repo) => {
       )
     }).timeout(30 * 1000)
 
+    it('export a directory one deep', (done) => {
+      const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
+
+      pull(
+        exporter(hash, ipldResolver, { maxDepth: 1}),
+        pull.collect((err, files) => {
+          expect(err).to.not.exist()
+          files.forEach(file => expect(file).to.have.property('hash'))
+
+          expect(
+            files.map((file) => file.path)
+          ).to.be.eql([
+            'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN',
+            'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt',
+            'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another',
+            'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1'
+          ])
+
+          pull(
+            pull.values(files),
+            pull.map((file) => Boolean(file.content)),
+            pull.collect((err, contents) => {
+              expect(err).to.not.exist()
+              expect(contents).to.be.eql([
+                false,
+                true,
+                false,
+                false
+              ])
+              done()
+            })
+          )
+        })
+      )
+    }).timeout(30 * 1000)
+
     it('returns an empty stream for dir', (done) => {
       const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
 

From 0761139ddb29f2732f118deb4f1edf163936fa08 Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Fri, 10 Nov 2017 17:42:59 +0000
Subject: [PATCH 07/10] fixed linting error

---
 test/exporter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/exporter.js b/test/exporter.js
index 3558724e..8daf5500 100644
--- a/test/exporter.js
+++ b/test/exporter.js
@@ -155,7 +155,7 @@ module.exports = (repo) => {
       const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
 
       pull(
-        exporter(hash, ipldResolver, { maxDepth: 1}),
+        exporter(hash, ipldResolver, { maxDepth: 1 }),
         pull.collect((err, files) => {
           expect(err).to.not.exist()
           files.forEach(file => expect(file).to.have.property('hash'))

From cc75c7019538a60f2ceeab39680e5a733cf49d9c Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Sat, 11 Nov 2017 15:51:26 +0000
Subject: [PATCH 08/10] exporter: exporting link size insteaf of node size to
 mimc go-ipfs

---
 src/exporter/dir-flat.js         | 2 +-
 src/exporter/dir-hamt-sharded.js | 2 +-
 src/exporter/file.js             | 4 ++--
 src/exporter/object.js           | 2 +-
 src/exporter/resolve.js          | 6 +++---
 test/builder-dir-sharding.js     | 2 +-
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/exporter/dir-flat.js b/src/exporter/dir-flat.js
index 46fa9323..9a75f6fa 100644
--- a/src/exporter/dir-flat.js
+++ b/src/exporter/dir-flat.js
@@ -6,7 +6,7 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, path, pathRest, resolve, dag, parent, depth) {
+function dirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
   const accepts = pathRest[0]
 
   const dir = {
diff --git a/src/exporter/dir-hamt-sharded.js b/src/exporter/dir-hamt-sharded.js
index 0f744756..6569d2a4 100644
--- a/src/exporter/dir-hamt-sharded.js
+++ b/src/exporter/dir-hamt-sharded.js
@@ -7,7 +7,7 @@ const cleanHash = require('./clean-multihash')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, path, pathRest, resolve, dag, parent, depth) {
+function shardedDirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
   let dir
   if (!parent || (parent.path !== path)) {
     dir = {
diff --git a/src/exporter/file.js b/src/exporter/file.js
index 134989d7..38259eb4 100644
--- a/src/exporter/file.js
+++ b/src/exporter/file.js
@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, path, pathRest, resolve, dag, parent, depth) => {
+module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -43,7 +43,7 @@ module.exports = (node, name, path, pathRest, resolve, dag, parent, depth) => {
     name: name,
     path: path,
     hash: node.multihash,
-    size: file.fileSize(),
+    size: size || file.fileSize(),
     type: 'file'
   }])
 }
diff --git a/src/exporter/object.js b/src/exporter/object.js
index 3f7f819f..f6383eab 100644
--- a/src/exporter/object.js
+++ b/src/exporter/object.js
@@ -3,7 +3,7 @@
 const CID = require('cids')
 const pull = require('pull-stream')
 
-module.exports = (node, name, path, pathRest, resolve, dag, parent, depth) => {
+module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth) => {
   let newNode
   if (pathRest.length) {
     const pathElem = pathRest[0]
diff --git a/src/exporter/resolve.js b/src/exporter/resolve.js
index 3ee01235..1aa8976f 100644
--- a/src/exporter/resolve.js
+++ b/src/exporter/resolve.js
@@ -48,17 +48,17 @@ function createResolver (dag, options, depth, parent) {
   )
 
   function resolveItem (node, item) {
-    return resolve(node, item.name, item.path, item.pathRest, dag, item.parent || parent, item.depth)
+    return resolve(node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth)
   }
 
-  function resolve (node, name, path, pathRest, dag, parentNode, depth) {
+  function resolve (node, name, path, pathRest, size, dag, parentNode, depth) {
     const type = typeOf(node)
     const nodeResolver = resolvers[type]
     if (!nodeResolver) {
       return pull.error(new Error('Unkown node type ' + type))
     }
     const resolveDeep = createResolver(dag, options, depth, node)
-    return nodeResolver(node, name, path, pathRest, resolveDeep, dag, parentNode, depth)
+    return nodeResolver(node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth)
   }
 }
 
diff --git a/test/builder-dir-sharding.js b/test/builder-dir-sharding.js
index 203e63c2..f2573ae3 100644
--- a/test/builder-dir-sharding.js
+++ b/test/builder-dir-sharding.js
@@ -91,7 +91,7 @@ module.exports = (repo) => {
             expect(nodes[0].path).to.be.eql(expectedHash)
             expect(mh.toB58String(nodes[0].hash)).to.be.eql(expectedHash)
             expect(nodes[1].path).to.be.eql(expectedHash + '/b')
-            expect(nodes[1].size).to.be.eql(21)
+            expect(nodes[1].size).to.be.eql(29)
             pull(
               nodes[1].content,
               pull.collect(collected)

From 262976bf770b9782d01fbeb6cc8538a25518c432 Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Sat, 11 Nov 2017 15:53:56 +0000
Subject: [PATCH 09/10] test: moving timeout def to the test top

---
 test/exporter.js | 15 ++++++++++-----
 test/importer.js |  6 ++++--
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/test/exporter.js b/test/exporter.js
index 8daf5500..c3510584 100644
--- a/test/exporter.js
+++ b/test/exporter.js
@@ -74,6 +74,7 @@ module.exports = (repo) => {
     })
 
     it('export a small file with links', (done) => {
+      this.timeout(30 * 1000)
       const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q'
       pull(
         exporter(hash, ipldResolver),
@@ -83,9 +84,10 @@ module.exports = (repo) => {
           fileEql(files[0], bigFile, done)
         })
       )
-    }).timeout(30 * 1000)
+    })
 
     it('export a small file with links using CID instead of multihash', (done) => {
+      this.timeout(30 * 1000)
       const cid = new CID('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q')
 
       pull(
@@ -96,9 +98,10 @@ module.exports = (repo) => {
           fileEql(files[0], bigFile, done)
         })
       )
-    }).timeout(30 * 1000)
+    })
 
     it('export a large file > 5mb', (done) => {
+      this.timeout(30 * 1000)
       const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE'
       pull(
         exporter(hash, ipldResolver),
@@ -109,9 +112,10 @@ module.exports = (repo) => {
           fileEql(files[0], null, done)
         })
       )
-    }).timeout(30 * 1000)
+    })
 
     it('export a directory', (done) => {
+      this.timeout(30 * 1000)
       const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
 
       pull(
@@ -149,9 +153,10 @@ module.exports = (repo) => {
           )
         })
       )
-    }).timeout(30 * 1000)
+    })
 
     it('export a directory one deep', (done) => {
+      this.timeout(30 * 1000)
       const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
 
       pull(
@@ -185,7 +190,7 @@ module.exports = (repo) => {
           )
         })
       )
-    }).timeout(30 * 1000)
+    })
 
     it('returns an empty stream for dir', (done) => {
       const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
diff --git a/test/importer.js b/test/importer.js
index 588d590b..cb1bbad8 100644
--- a/test/importer.js
+++ b/test/importer.js
@@ -299,6 +299,7 @@ module.exports = (repo) => {
       })
 
       it('file bigger than a single chunk', (done) => {
+        this.timeout(60 * 1000)
         pull(
           pull.values([{
             path: '1.2MiB.txt',
@@ -311,9 +312,10 @@ module.exports = (repo) => {
             done()
           })
         )
-      }).timeout(60 * 1000)
+      })
 
       it('file bigger than a single chunk inside a dir', (done) => {
+        this.timeout(60 * 1000)
         pull(
           pull.values([{
             path: 'foo-big/1.2MiB.txt',
@@ -331,7 +333,7 @@ module.exports = (repo) => {
             done()
           })
         )
-      }).timeout(60 * 1000)
+      })
 
       it('empty directory', (done) => {
         pull(

From 5aa7731285123c108329fd631009ea56ec0fbb4e Mon Sep 17 00:00:00 2001
From: Pedro Teixeira <i@pgte.me>
Date: Sun, 12 Nov 2017 10:16:53 +0000
Subject: [PATCH 10/10] tests: fixed this.timeout because arrow functions

---
 test/exporter.js | 10 +++++-----
 test/importer.js |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/exporter.js b/test/exporter.js
index c3510584..39f72f7f 100644
--- a/test/exporter.js
+++ b/test/exporter.js
@@ -73,7 +73,7 @@ module.exports = (repo) => {
       )
     })
 
-    it('export a small file with links', (done) => {
+    it('export a small file with links', function (done) {
       this.timeout(30 * 1000)
       const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q'
       pull(
@@ -86,7 +86,7 @@ module.exports = (repo) => {
       )
     })
 
-    it('export a small file with links using CID instead of multihash', (done) => {
+    it('export a small file with links using CID instead of multihash', function (done) {
       this.timeout(30 * 1000)
       const cid = new CID('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q')
 
@@ -100,7 +100,7 @@ module.exports = (repo) => {
       )
     })
 
-    it('export a large file > 5mb', (done) => {
+    it('export a large file > 5mb', function (done) {
       this.timeout(30 * 1000)
       const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE'
       pull(
@@ -114,7 +114,7 @@ module.exports = (repo) => {
       )
     })
 
-    it('export a directory', (done) => {
+    it('export a directory', function (done) {
       this.timeout(30 * 1000)
       const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
 
@@ -155,7 +155,7 @@ module.exports = (repo) => {
       )
     })
 
-    it('export a directory one deep', (done) => {
+    it('export a directory one deep', function (done) {
       this.timeout(30 * 1000)
       const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
 
diff --git a/test/importer.js b/test/importer.js
index cb1bbad8..468f1299 100644
--- a/test/importer.js
+++ b/test/importer.js
@@ -298,7 +298,7 @@ module.exports = (repo) => {
         }
       })
 
-      it('file bigger than a single chunk', (done) => {
+      it('file bigger than a single chunk', function (done) {
         this.timeout(60 * 1000)
         pull(
           pull.values([{
@@ -314,7 +314,7 @@ module.exports = (repo) => {
         )
       })
 
-      it('file bigger than a single chunk inside a dir', (done) => {
+      it('file bigger than a single chunk inside a dir', function (done) {
         this.timeout(60 * 1000)
         pull(
           pull.values([{