Skip to content

Commit f951513

Browse files
eduardoboucaseduardoboucas
authored
feat!: return AsyncIterator for list() (#102)
1 parent 51fd622 commit f951513

File tree

4 files changed

+241
-157
lines changed

4 files changed

+241
-157
lines changed

‎README.md

+24-1
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ not there was an object to delete.
278278
await store.delete('my-key')
279279
```
280280

281-
### `list(options?: { cursor?: string, directories?: boolean, paginate?: boolean. prefix?: string }): Promise<{ blobs: BlobResult[], directories: string[] }>`
281+
### `list(options?: { directories?: boolean, paginate?: boolean. prefix?: string }): Promise<{ blobs: BlobResult[], directories: string[] }> | AsyncIterable<{ blobs: BlobResult[], directories: string[] }>`
282282

283283
Returns a list of blobs in a given store.
284284

@@ -355,6 +355,29 @@ console.log(directories)
355355
Note that we're only interested in entries under the `cats` directory, which is why we're using a trailing slash.
356356
Without it, other keys like `catsuit` would also match.
357357

358+
For performance reasons, the server groups results into pages of up to 1,000 entries. By default, the `list()` method
359+
automatically retrieves all pages, meaning you'll always get the full list of results.
360+
361+
If you'd like to handle this pagination manually, you can supply the `paginate` parameter, which makes `list()` return
362+
an [`AsyncIterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncIterator).
363+
364+
```javascript
365+
const blobs = []
366+
367+
for await (const entry of store.list({ paginate: true })) {
368+
blobs.push(...entry.blobs)
369+
}
370+
371+
// [
372+
// { etag: "etag1", key: "cats/garfield.jpg" },
373+
// { etag: "etag2", key: "cats/tom.jpg" },
374+
// { etag: "etag3", key: "mice/jerry.jpg" },
375+
// { etag: "etag4", key: "mice/mickey.jpg" },
376+
// { etag: "etag5", key: "pink-panther.jpg" },
377+
// ]
378+
console.log(blobs)
379+
```
380+
358381
## Contributing
359382

360383
Contributions are welcome! If you encounter any issues or have suggestions for improvements, please open an issue or

‎src/list.test.ts

+139-71
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { describe, test, expect, beforeAll, afterEach } from 'vitest'
66
import { MockFetch } from '../test/mock_fetch.js'
77

88
import { getStore } from './main.js'
9+
import type { ListResult } from './store.js'
910

1011
beforeAll(async () => {
1112
if (semver.lt(nodeVersion, '18.0.0')) {
@@ -171,7 +172,7 @@ describe('list', () => {
171172
next_cursor: 'cursor_2',
172173
}),
173174
),
174-
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&directories=true&context=${storeName}`,
175+
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?directories=true&cursor=cursor_1&context=${storeName}`,
175176
})
176177
.get({
177178
headers: { authorization: `Bearer ${apiToken}` },
@@ -188,7 +189,7 @@ describe('list', () => {
188189
directories: ['dir3'],
189190
}),
190191
),
191-
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&directories=true&context=${storeName}`,
192+
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?directories=true&cursor=cursor_2&context=${storeName}`,
192193
})
193194
.get({
194195
headers: { authorization: `Bearer ${apiToken}` },
@@ -279,30 +280,47 @@ describe('list', () => {
279280
expect(mockStore.fulfilled).toBeTruthy()
280281
})
281282

282-
test('Paginates manually with `cursor` if `paginate: false`', async () => {
283-
const mockStore = new MockFetch().get({
284-
headers: { authorization: `Bearer ${apiToken}` },
285-
response: new Response(
286-
JSON.stringify({
287-
blobs: [
288-
{
289-
etag: 'etag3',
290-
key: 'key3',
291-
size: 3,
292-
last_modified: '2023-07-18T12:59:06Z',
293-
},
294-
{
295-
etag: 'etag4',
296-
key: 'key4',
297-
size: 4,
298-
last_modified: '2023-07-18T12:59:06Z',
299-
},
300-
],
301-
next_cursor: 'cursor_2',
302-
}),
303-
),
304-
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&context=${storeName}`,
305-
})
283+
test('Returns an `AsyncIterator` if `paginate: true`', async () => {
284+
const mockStore = new MockFetch()
285+
.get({
286+
headers: { authorization: `Bearer ${apiToken}` },
287+
response: new Response(
288+
JSON.stringify({
289+
blobs: [
290+
{
291+
etag: 'etag1',
292+
key: 'key1',
293+
size: 1,
294+
last_modified: '2023-07-18T12:59:06Z',
295+
},
296+
{
297+
etag: 'etag2',
298+
key: 'key2',
299+
size: 2,
300+
last_modified: '2023-07-18T12:59:06Z',
301+
},
302+
],
303+
next_cursor: 'cursor_2',
304+
}),
305+
),
306+
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?context=${storeName}`,
307+
})
308+
.get({
309+
headers: { authorization: `Bearer ${apiToken}` },
310+
response: new Response(
311+
JSON.stringify({
312+
blobs: [
313+
{
314+
etag: 'etag3',
315+
key: 'key3',
316+
size: 3,
317+
last_modified: '2023-07-18T12:59:06Z',
318+
},
319+
],
320+
}),
321+
),
322+
url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&context=${storeName}`,
323+
})
306324

307325
globalThis.fetch = mockStore.fetch
308326

@@ -311,15 +329,20 @@ describe('list', () => {
311329
token: apiToken,
312330
siteID,
313331
})
332+
const result: ListResult = {
333+
blobs: [],
334+
directories: [],
335+
}
314336

315-
const { blobs } = await store.list({
316-
cursor: 'cursor_1',
317-
paginate: false,
318-
})
337+
for await (const entry of store.list({ paginate: true })) {
338+
result.blobs.push(...entry.blobs)
339+
result.directories.push(...entry.directories)
340+
}
319341

320-
expect(blobs).toEqual([
342+
expect(result.blobs).toEqual([
343+
{ etag: 'etag1', key: 'key1' },
344+
{ etag: 'etag2', key: 'key2' },
321345
{ etag: 'etag3', key: 'key3' },
322-
{ etag: 'etag4', key: 'key4' },
323346
])
324347
expect(mockStore.fulfilled).toBeTruthy()
325348
})
@@ -346,7 +369,7 @@ describe('list', () => {
346369
last_modified: '2023-07-18T12:59:06Z',
347370
},
348371
],
349-
directories: ['dir1'],
372+
directories: [],
350373
next_cursor: 'cursor_1',
351374
}),
352375
),
@@ -370,7 +393,7 @@ describe('list', () => {
370393
last_modified: '2023-07-18T12:59:06Z',
371394
},
372395
],
373-
directories: ['dir2'],
396+
directories: [],
374397
next_cursor: 'cursor_2',
375398
}),
376399
),
@@ -388,7 +411,7 @@ describe('list', () => {
388411
last_modified: '2023-07-18T12:59:06Z',
389412
},
390413
],
391-
directories: ['dir3'],
414+
directories: [],
392415
}),
393416
),
394417
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2`,
@@ -430,15 +453,12 @@ describe('list', () => {
430453
{ etag: 'etag5', key: 'key5' },
431454
])
432455

433-
// @ts-expect-error `directories` is not part of the return type
434-
expect(root.directories).toBe(undefined)
456+
expect(root.directories).toEqual([])
435457

436458
const directory = await store.list({ prefix: 'dir2/' })
437459

438460
expect(directory.blobs).toEqual([{ etag: 'etag6', key: 'key6' }])
439-
440-
// @ts-expect-error `directories` is not part of the return type
441-
expect(directory.directories).toBe(undefined)
461+
expect(directory.directories).toEqual([])
442462

443463
expect(mockStore.fulfilled).toBeTruthy()
444464
})
@@ -491,7 +511,7 @@ describe('list', () => {
491511
next_cursor: 'cursor_2',
492512
}),
493513
),
494-
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1&directories=true`,
514+
url: `${edgeURL}/${siteID}/${storeName}?directories=true&cursor=cursor_1`,
495515
})
496516
.get({
497517
headers: { authorization: `Bearer ${edgeToken}` },
@@ -508,7 +528,7 @@ describe('list', () => {
508528
directories: ['dir3'],
509529
}),
510530
),
511-
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2&directories=true`,
531+
url: `${edgeURL}/${siteID}/${storeName}?directories=true&cursor=cursor_2`,
512532
})
513533
.get({
514534
headers: { authorization: `Bearer ${edgeToken}` },
@@ -601,30 +621,70 @@ describe('list', () => {
601621
expect(mockStore.fulfilled).toBeTruthy()
602622
})
603623

604-
test('Paginates manually with `cursor` if `paginate: false`', async () => {
605-
const mockStore = new MockFetch().get({
606-
headers: { authorization: `Bearer ${edgeToken}` },
607-
response: new Response(
608-
JSON.stringify({
609-
blobs: [
610-
{
611-
etag: 'etag3',
612-
key: 'key3',
613-
size: 3,
614-
last_modified: '2023-07-18T12:59:06Z',
615-
},
616-
{
617-
etag: 'etag4',
618-
key: 'key4',
619-
size: 4,
620-
last_modified: '2023-07-18T12:59:06Z',
621-
},
622-
],
623-
next_cursor: 'cursor_2',
624-
}),
625-
),
626-
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1`,
627-
})
624+
test('Returns an `AsyncIterator` if `paginate: true`', async () => {
625+
const mockStore = new MockFetch()
626+
.get({
627+
headers: { authorization: `Bearer ${edgeToken}` },
628+
response: new Response(
629+
JSON.stringify({
630+
blobs: [
631+
{
632+
etag: 'etag1',
633+
key: 'key1',
634+
size: 1,
635+
last_modified: '2023-07-18T12:59:06Z',
636+
},
637+
{
638+
etag: 'etag2',
639+
key: 'key2',
640+
size: 2,
641+
last_modified: '2023-07-18T12:59:06Z',
642+
},
643+
],
644+
next_cursor: 'cursor_2',
645+
}),
646+
),
647+
url: `${edgeURL}/${siteID}/${storeName}`,
648+
})
649+
.get({
650+
headers: { authorization: `Bearer ${edgeToken}` },
651+
response: new Response(
652+
JSON.stringify({
653+
blobs: [
654+
{
655+
etag: 'etag3',
656+
key: 'key3',
657+
size: 3,
658+
last_modified: '2023-07-18T12:59:06Z',
659+
},
660+
{
661+
etag: 'etag4',
662+
key: 'key4',
663+
size: 4,
664+
last_modified: '2023-07-18T12:59:06Z',
665+
},
666+
],
667+
next_cursor: 'cursor_3',
668+
}),
669+
),
670+
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2`,
671+
})
672+
.get({
673+
headers: { authorization: `Bearer ${edgeToken}` },
674+
response: new Response(
675+
JSON.stringify({
676+
blobs: [
677+
{
678+
etag: 'etag5',
679+
key: 'key5',
680+
size: 5,
681+
last_modified: '2023-07-18T12:59:06Z',
682+
},
683+
],
684+
}),
685+
),
686+
url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_3`,
687+
})
628688

629689
globalThis.fetch = mockStore.fetch
630690

@@ -634,16 +694,24 @@ describe('list', () => {
634694
token: edgeToken,
635695
siteID,
636696
})
697+
const result: ListResult = {
698+
blobs: [],
699+
directories: [],
700+
}
637701

638-
const { blobs } = await store.list({
639-
cursor: 'cursor_1',
640-
paginate: false,
641-
})
702+
for await (const entry of store.list({ paginate: true })) {
703+
result.blobs.push(...entry.blobs)
704+
result.directories.push(...entry.directories)
705+
}
642706

643-
expect(blobs).toEqual([
707+
expect(result.blobs).toEqual([
708+
{ etag: 'etag1', key: 'key1' },
709+
{ etag: 'etag2', key: 'key2' },
644710
{ etag: 'etag3', key: 'key3' },
645711
{ etag: 'etag4', key: 'key4' },
712+
{ etag: 'etag5', key: 'key5' },
646713
])
714+
expect(result.directories).toEqual([])
647715
expect(mockStore.fulfilled).toBeTruthy()
648716
})
649717
})

0 commit comments

Comments
 (0)