From 03ac208711a81fe6745f60b211287d211f861451 Mon Sep 17 00:00:00 2001
From: Brandon Ramirez <brandon.ramirez@live.com>
Date: Tue, 24 Apr 2018 22:10:19 -0700
Subject: [PATCH] Use generators in ETL mapping example to avoid memory issues

---
 .../OriginListing_old/ETL/ETL-mapping.js      | 41 ++++++++-----------
 1 file changed, 16 insertions(+), 25 deletions(-)

diff --git a/specs/data-source-definition/examples/OriginListing_old/ETL/ETL-mapping.js b/specs/data-source-definition/examples/OriginListing_old/ETL/ETL-mapping.js
index c8d3acd..bcbe1b3 100644
--- a/specs/data-source-definition/examples/OriginListing_old/ETL/ETL-mapping.js
+++ b/specs/data-source-definition/examples/OriginListing_old/ETL/ETL-mapping.js
@@ -7,34 +7,25 @@ import { map, range } from 'lodash/fp'
  * @param  {Object} ipfs     A client for interacting with an IPFS storage backend.
  * @param  {Object} options  An object which may contain clients for storage adapters.
  */
-export const extract = (data, { adapters }) => {
+export function* extract(data, { adapters }) {
   const { ipfs } = adapters
   const originListing = data
+  // Need to pretend that entityCount can be arbitrarily large
   const entityCount = originListing.listingsLength()
-  const entities = range(0, entityCount)
-    .map(value => originListing.getListing(value))
-    .map(([index, lister, ipfsHash, price, unitsAvailable]) => ({
-      index,
-      lister,
-      ipfsHash,
-      price,
-      unitsAvailable,
-    }))
-  return Promise.all(
-    entities.map(entity =>
-      // OriginListing contract uses truncated IPFS hashes
-      ipfs.block
-        .get(`Qm${entity.ipfsHash}`)
-        .then(data => JSON.parse(data))
-        .then(ipfsObject => ({
-          index: entity.index,
-          lister: entity.lister,
-          price: entity.price,
-          unitsAvailable: unitsAvailable,
-          listingData: ipfsObject.data,
-        })),
-    ),
-  )
+  for (let i = 0; i < entityCount; i++) {
+    let listing = originListing.getListing(i)
+    let [index, lister, ipfsHash, price, unitsAvailable] = listing
+    yield ipfs.block
+      .get(`Qm${entity.ipfsHash}`)
+      .then(data => JSON.parse(data))
+      .then(ipfsObject => ({
+        index: entity.index,
+        lister: entity.lister,
+        price: entity.price,
+        unitsAvailable: unitsAvailable,
+        listingData: ipfsObject.data,
+      }))
+  }
 }
 
 /**