From 23fde7432509afdee4bf1167600f6322d4ec084e Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 13:41:30 +1200 Subject: [PATCH 01/14] Duplicate blockstore-fs. --- packages/blockstore-filestore/CHANGELOG.md | 52 ++++ packages/blockstore-filestore/LICENSE | 4 + packages/blockstore-filestore/LICENSE-APACHE | 5 + packages/blockstore-filestore/LICENSE-MIT | 19 ++ packages/blockstore-filestore/README.md | 53 ++++ .../benchmarks/encoding/package.json | 18 ++ .../benchmarks/encoding/src/README.md | 31 +++ .../benchmarks/encoding/src/index.ts | 73 +++++ .../benchmarks/encoding/tsconfig.json | 13 + packages/blockstore-filestore/package.json | 182 +++++++++++++ packages/blockstore-filestore/src/index.ts | 254 ++++++++++++++++++ packages/blockstore-filestore/src/sharding.ts | 117 ++++++++ .../blockstore-filestore/test/index.spec.ts | 161 +++++++++++ .../test/sharding.spec.ts | 109 ++++++++ packages/blockstore-filestore/tsconfig.json | 24 ++ 15 files changed, 1115 insertions(+) create mode 100644 packages/blockstore-filestore/CHANGELOG.md create mode 100644 packages/blockstore-filestore/LICENSE create mode 100644 packages/blockstore-filestore/LICENSE-APACHE create mode 100644 packages/blockstore-filestore/LICENSE-MIT create mode 100644 packages/blockstore-filestore/README.md create mode 100644 packages/blockstore-filestore/benchmarks/encoding/package.json create mode 100644 packages/blockstore-filestore/benchmarks/encoding/src/README.md create mode 100644 packages/blockstore-filestore/benchmarks/encoding/src/index.ts create mode 100644 packages/blockstore-filestore/benchmarks/encoding/tsconfig.json create mode 100644 packages/blockstore-filestore/package.json create mode 100644 packages/blockstore-filestore/src/index.ts create mode 100644 packages/blockstore-filestore/src/sharding.ts create mode 100644 packages/blockstore-filestore/test/index.spec.ts create mode 100644 packages/blockstore-filestore/test/sharding.spec.ts create mode 100644 packages/blockstore-filestore/tsconfig.json diff --git a/packages/blockstore-filestore/CHANGELOG.md b/packages/blockstore-filestore/CHANGELOG.md new file mode 100644 index 00000000..dd7434c3 --- /dev/null +++ b/packages/blockstore-filestore/CHANGELOG.md @@ -0,0 +1,52 @@ +## [blockstore-fs-v1.1.3](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.2...blockstore-fs-v1.1.3) (2023-06-03) + + +### Documentation + +* fix capitalization of import ([#226](https://github.com/ipfs/js-stores/issues/226)) ([837221a](https://github.com/ipfs/js-stores/commit/837221aff3ef4d217063eb17953aff03764e7600)) + +## [blockstore-fs-v1.1.2](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.1...blockstore-fs-v1.1.2) (2023-06-03) + + +### Dependencies + +* bump aegir from 38.1.8 to 39.0.9 ([#225](https://github.com/ipfs/js-stores/issues/225)) ([d0f301b](https://github.com/ipfs/js-stores/commit/d0f301b1243a0f4f692011449567b51b2706e70f)) + +## [blockstore-fs-v1.1.1](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.0...blockstore-fs-v1.1.1) (2023-03-31) + + +### Trivial Changes + +* rename master to main ([#200](https://github.com/ipfs/js-stores/issues/200)) ([f85d719](https://github.com/ipfs/js-stores/commit/f85d719b711cd60237bdaa6a0bcd418e69a98598)) + + +### Dependencies + +* update all it-* deps ([#213](https://github.com/ipfs/js-stores/issues/213)) ([e963497](https://github.com/ipfs/js-stores/commit/e963497fdb33e61e2fe702866abbd42fba648fee)) + +## [blockstore-fs-v1.1.0](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.0.1...blockstore-fs-v1.1.0) (2023-03-23) + + +### Features + +* add all blockstore and datastore implementations ([#197](https://github.com/ipfs/js-stores/issues/197)) ([0d85128](https://github.com/ipfs/js-stores/commit/0d851286d48c357b07df3f7419c1e903ed0e7fac)) + +## [1.0.1](https://github.com/ipfs/js-blockstore-fs/compare/v1.0.0...v1.0.1) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#1](https://github.com/ipfs/js-blockstore-fs/issues/1)) ([88fa91c](https://github.com/ipfs/js-blockstore-fs/commit/88fa91cb1405ed66f053ed265c1690ac0ad22214)) + +## 1.0.0 (2023-03-13) + + +### Bug Fixes + +* update ci build files ([bdc1881](https://github.com/ipfs/js-blockstore-fs/commit/bdc18810e6d63ffdbf6fc6617757aaa96b0ba82c)) + + +### Trivial Changes + +* add missing dep ([f5ba415](https://github.com/ipfs/js-blockstore-fs/commit/f5ba41536816f08eb1b97f298091c7221c6c9360)) +* initial import ([7a576cb](https://github.com/ipfs/js-blockstore-fs/commit/7a576cbad5696ad396c0cd2d557edf71d624a860)) diff --git a/packages/blockstore-filestore/LICENSE b/packages/blockstore-filestore/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-filestore/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-filestore/LICENSE-APACHE b/packages/blockstore-filestore/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-filestore/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-filestore/LICENSE-MIT b/packages/blockstore-filestore/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-filestore/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-filestore/README.md b/packages/blockstore-filestore/README.md new file mode 100644 index 00000000..e46e8afa --- /dev/null +++ b/packages/blockstore-filestore/README.md @@ -0,0 +1,53 @@ +# blockstore-fs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Blockstore implementation with file system backend + +## Table of contents + +- [Install](#install) +- [Usage](#usage) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) + +## Install + +```console +$ npm i blockstore-fs +``` + +## Usage + +```js +import { FsBlockstore } from 'blockstore-fs' + +const store = new FsBlockstore('path/to/store') +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-filestore/benchmarks/encoding/package.json b/packages/blockstore-filestore/benchmarks/encoding/package.json new file mode 100644 index 00000000..00104d4f --- /dev/null +++ b/packages/blockstore-filestore/benchmarks/encoding/package.json @@ -0,0 +1,18 @@ +{ + "name": "benchmarks-encoding", + "version": "1.0.0", + "main": "index.js", + "private": true, + "type": "module", + "scripts": { + "clean": "aegir clean", + "build": "aegir build --bundle false", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "start": "npm run build && node dist/src/index.js" + }, + "devDependencies": { + "multiformats": "^11.0.1", + "tinybench": "^2.4.0" + } +} diff --git a/packages/blockstore-filestore/benchmarks/encoding/src/README.md b/packages/blockstore-filestore/benchmarks/encoding/src/README.md new file mode 100644 index 00000000..3caf1e9e --- /dev/null +++ b/packages/blockstore-filestore/benchmarks/encoding/src/README.md @@ -0,0 +1,31 @@ +# Encoding Benchmark + +Multiformats ships a number of base encoding algorithms. This module has no strong opinion +on which is best, as long as it is case insensitive, so benchmark them to choose the fastest. + +At the time of writing `base8` is the fastest, followed other alorithms using `rfc4648` encoding +internally in `multiformats` (e.g. `base16`, `base32`), and finally anything using `baseX` encoding. + +We choose base32upper which uses `rfc4648` because it has a longer alphabet so will shard better. + +## Usage + +```console +$ npm i +$ npm start + +> benchmarks-gc@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-gc@1.0.0 build +> aegir build --bundle false + +[14:51:28] tsc [started] +[14:51:33] tsc [completed] +generating Ed25519 keypair... +┌─────────┬────────────────┬─────────┬───────────┬──────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ +├─────────┼────────────────┼─────────┼───────────┼──────┤ +//... results here +``` diff --git a/packages/blockstore-filestore/benchmarks/encoding/src/index.ts b/packages/blockstore-filestore/benchmarks/encoding/src/index.ts new file mode 100644 index 00000000..40fc0905 --- /dev/null +++ b/packages/blockstore-filestore/benchmarks/encoding/src/index.ts @@ -0,0 +1,73 @@ +import { base10 } from 'multiformats/bases/base10' +import { base16upper } from 'multiformats/bases/base16' +import { base256emoji } from 'multiformats/bases/base256emoji' +import { base32, base32upper, base32hexupper, base32z } from 'multiformats/bases/base32' +import { base36, base36upper } from 'multiformats/bases/base36' +import { base8 } from 'multiformats/bases/base8' +import { CID } from 'multiformats/cid' +import { Bench } from 'tinybench' + +const RESULT_PRECISION = 2 + +const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + +async function main (): Promise { + const suite = new Bench() + suite.add('base8', () => { + base8.encode(cid.bytes) + }) + suite.add('base10', () => { + base10.encode(cid.bytes) + }) + suite.add('base16upper', () => { + base16upper.encode(cid.bytes) + }) + suite.add('base32', () => { + base32.encode(cid.bytes) + }) + suite.add('base32upper', () => { + base32upper.encode(cid.bytes) + }) + suite.add('base32hexupper', () => { + base32hexupper.encode(cid.bytes) + }) + suite.add('base32z', () => { + base32z.encode(cid.bytes) + }) + suite.add('base36', () => { + base36.encode(cid.bytes) + }) + suite.add('base36upper', () => { + base36upper.encode(cid.bytes) + }) + suite.add('base256emoji', () => { + base256emoji.encode(cid.bytes) + }) + + await suite.run() + + console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console + const resultA = a.result?.hz ?? 0 + const resultB = b.result?.hz ?? 0 + + if (resultA === resultB) { + return 0 + } + + if (resultA < resultB) { + return 1 + } + + return -1 + }).map(({ name, result }) => ({ + Implementation: name, + 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), + 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), + runs: result?.samples.length + }))) +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json b/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json new file mode 100644 index 00000000..fee64009 --- /dev/null +++ b/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist", + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"] + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json new file mode 100644 index 00000000..a7942fb4 --- /dev/null +++ b/packages/blockstore-filestore/package.json @@ -0,0 +1,182 @@ +{ + "name": "blockstore-fs", + "version": "1.1.3", + "description": "Blockstore implementation with file system backend", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-fs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "fs", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./sharding": { + "types": "./dist/src/sharding.d.ts", + "import": "./dist/src/sharding.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module", + "project": [ + "tsconfig.json", + "benchmarks/encoding/tsconfig.json" + ] + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build --bundle false", + "release": "aegir release", + "test": "aegir test -t node -t electron-main", + "test:node": "aegir test -t node", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "blockstore-core": "^4.0.0", + "fast-write-atomic": "^0.2.0", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "it-glob": "^2.0.1", + "it-map": "^3.0.1", + "it-parallel-batch": "^3.0.0", + "multiformats": "^11.0.2" + }, + "devDependencies": { + "aegir": "^39.0.9", + "interface-blockstore-tests": "^6.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts new file mode 100644 index 00000000..06471475 --- /dev/null +++ b/packages/blockstore-filestore/src/index.ts @@ -0,0 +1,254 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { promisify } from 'node:util' +import { + Errors +} from 'blockstore-core' +// @ts-expect-error no types +import fwa from 'fast-write-atomic' +import glob from 'it-glob' +import map from 'it-map' +import parallelBatch from 'it-parallel-batch' +import { NextToLast, type ShardingStrategy } from './sharding.js' +import type { Blockstore, Pair } from 'interface-blockstore' +import type { AwaitIterable } from 'interface-store' +import type { CID } from 'multiformats/cid' + +const writeAtomic = promisify(fwa) + +/** + * Write a file atomically + */ +async function writeFile (file: string, contents: Uint8Array): Promise { + try { + await writeAtomic(file, contents) + } catch (err: any) { + if (err.code === 'EPERM' && err.syscall === 'rename') { + // fast-write-atomic writes a file to a temp location before renaming it. + // On Windows, if the final file already exists this error is thrown. + // No such error is thrown on Linux/Mac + // Make sure we can read & write to this file + await fs.access(file, fs.constants.F_OK | fs.constants.W_OK) + + // The file was created by another context - this means there were + // attempts to write the same block by two different function calls + return + } + + throw err + } +} + +export interface FsBlockstoreInit { + /** + * If true and the passed blockstore location does not exist, create + * it on startup. default: true + */ + createIfMissing?: boolean + + /** + * If true and the passed blockstore location exists on startup, throw + * an error. default: false + */ + errorIfExists?: boolean + + /** + * The file extension to use when storing blocks. default: '.data' + */ + extension?: string + + /** + * How many blocks to put in parallel when `.putMany` is called. + * default: 50 + */ + putManyConcurrency?: number + + /** + * How many blocks to read in parallel when `.getMany` is called. + * default: 50 + */ + getManyConcurrency?: number + + /** + * How many blocks to delete in parallel when `.deleteMany` is called. + * default: 50 + */ + deleteManyConcurrency?: number + + /** + * Control how CIDs map to paths and back + */ + shardingStrategy?: ShardingStrategy +} + +/** + * A blockstore backed by the file system + */ +export class FsBlockstore implements Blockstore { + public path: string + private readonly createIfMissing: boolean + private readonly errorIfExists: boolean + private readonly putManyConcurrency: number + private readonly getManyConcurrency: number + private readonly deleteManyConcurrency: number + private readonly shardingStrategy: ShardingStrategy + + constructor (location: string, init: FsBlockstoreInit = {}) { + this.path = path.resolve(location) + this.createIfMissing = init.createIfMissing ?? true + this.errorIfExists = init.errorIfExists ?? false + this.deleteManyConcurrency = init.deleteManyConcurrency ?? 50 + this.getManyConcurrency = init.getManyConcurrency ?? 50 + this.putManyConcurrency = init.putManyConcurrency ?? 50 + this.shardingStrategy = init.shardingStrategy ?? new NextToLast() + } + + async open (): Promise { + try { + await fs.access(this.path, fs.constants.F_OK | fs.constants.W_OK) + + if (this.errorIfExists) { + throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} already exists`)) + } + } catch (err: any) { + if (err.code === 'ENOENT') { + if (this.createIfMissing) { + await fs.mkdir(this.path, { recursive: true }) + return + } else { + throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} does not exist`)) + } + } + + throw err + } + } + + async close (): Promise { + await Promise.resolve() + } + + async put (key: CID, val: Uint8Array): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + if (dir != null && dir !== '') { + await fs.mkdir(path.join(this.path, dir), { + recursive: true + }) + } + + await writeFile(path.join(this.path, dir, file), val) + + return key + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async * putMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, ({ cid, block }) => { + return async () => { + await this.put(cid, block) + + return cid + } + }), + this.putManyConcurrency + ) + } + + async get (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + return await fs.readFile(path.join(this.path, dir, file)) + } catch (err: any) { + throw Errors.notFoundError(err) + } + } + + async * getMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, key => { + return async () => { + return { + cid: key, + block: await this.get(key) + } + } + }), + this.getManyConcurrency + ) + } + + async delete (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + await fs.unlink(path.join(this.path, dir, file)) + } catch (err: any) { + if (err.code === 'ENOENT') { + return + } + + throw Errors.deleteFailedError(err) + } + } + + async * deleteMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, key => { + return async () => { + await this.delete(key) + + return key + } + }), + this.deleteManyConcurrency + ) + } + + /** + * Check for the existence of the given key + */ + async has (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + await fs.access(path.join(this.path, dir, file)) + } catch (err: any) { + return false + } + return true + } + + async * getAll (): AsyncIterable { + const pattern = `**/*${this.shardingStrategy.extension}` + .split(path.sep) + .join('/') + const files = glob(this.path, pattern, { + absolute: true + }) + + for await (const file of files) { + try { + const buf = await fs.readFile(file) + + const pair: Pair = { + cid: this.shardingStrategy.decode(file), + block: buf + } + + yield pair + } catch (err: any) { + // if keys are removed from the datastore while the query is + // running, we may encounter missing files. + if (err.code !== 'ENOENT') { + throw err + } + } + } + } +} diff --git a/packages/blockstore-filestore/src/sharding.ts b/packages/blockstore-filestore/src/sharding.ts new file mode 100644 index 00000000..10d65169 --- /dev/null +++ b/packages/blockstore-filestore/src/sharding.ts @@ -0,0 +1,117 @@ +import path from 'node:path' +import { base32upper } from 'multiformats/bases/base32' +import { CID } from 'multiformats/cid' +import type { MultibaseCodec } from 'multiformats/bases/interface' + +export interface ShardingStrategy { + extension: string + encode: (cid: CID) => { dir: string, file: string } + decode: (path: string) => CID +} + +export interface NextToLastInit { + /** + * The file extension to use. default: '.data' + */ + extension?: string + + /** + * How many characters to take from the end of the CID. default: 2 + */ + prefixLength?: number + + /** + * The multibase codec to use - nb. should be case insensitive. + * default: base32upper + */ + base?: MultibaseCodec +} + +/** + * A sharding strategy that takes the last few characters of a multibase encoded + * CID and uses them as the directory to store the block in. This prevents + * storing all blocks in a single directory which would overwhelm most + * filesystems. + */ +export class NextToLast implements ShardingStrategy { + public extension: string + private readonly prefixLength: number + private readonly base: MultibaseCodec + + constructor (init: NextToLastInit = {}) { + this.extension = init.extension ?? '.data' + this.prefixLength = init.prefixLength ?? 2 + this.base = init.base ?? base32upper + } + + encode (cid: CID): { dir: string, file: string } { + const str = this.base.encoder.encode(cid.multihash.bytes) + const prefix = str.substring(str.length - this.prefixLength) + + return { + dir: prefix, + file: `${str}${this.extension}` + } + } + + decode (str: string): CID { + let fileName = path.basename(str) + + if (fileName.endsWith(this.extension)) { + fileName = fileName.substring(0, fileName.length - this.extension.length) + } + + return CID.decode(this.base.decoder.decode(fileName)) + } +} + +export interface FlatDirectoryInit { + /** + * The file extension to use. default: '.data' + */ + extension?: string + + /** + * How many characters to take from the end of the CID. default: 2 + */ + prefixLength?: number + + /** + * The multibase codec to use - nb. should be case insensitive. + * default: base32padupper + */ + base?: MultibaseCodec +} + +/** + * A sharding strategy that does not do any sharding and stores all files + * in one directory. Only for testing, do not use in production. + */ +export class FlatDirectory implements ShardingStrategy { + public extension: string + private readonly base: MultibaseCodec + + constructor (init: NextToLastInit = {}) { + this.extension = init.extension ?? '.data' + this.base = init.base ?? base32upper + } + + encode (cid: CID): { dir: string, file: string } { + const str = this.base.encoder.encode(cid.multihash.bytes) + + return { + dir: '', + file: `${str}${this.extension}` + } + } + + decode (str: string): CID { + let fileName = path.basename(str) + + if (fileName.endsWith(this.extension)) { + fileName = fileName.substring(0, fileName.length - this.extension.length) + } + + return CID.decode(this.base.decoder.decode(fileName)) + } +} diff --git a/packages/blockstore-filestore/test/index.spec.ts b/packages/blockstore-filestore/test/index.spec.ts new file mode 100644 index 00000000..25e0ebc3 --- /dev/null +++ b/packages/blockstore-filestore/test/index.spec.ts @@ -0,0 +1,161 @@ +/* eslint-env mocha */ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { expect } from 'aegir/chai' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' +import { base256emoji } from 'multiformats/bases/base256emoji' +import { CID } from 'multiformats/cid' +import { FsBlockstore } from '../src/index.js' +import { FlatDirectory, NextToLast } from '../src/sharding.js' + +const utf8Encoder = new TextEncoder() + +describe('FsBlockstore', () => { + describe('construction', () => { + it('defaults - folder missing', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await expect( + (async () => { + const fs = new FsBlockstore(dir) + await fs.open() + await fs.close() + })() + ).to.eventually.be.undefined() + }) + + it('defaults - folder exists', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await fs.mkdir(dir, { + recursive: true + }) + await expect( + (async () => { + const fs = new FsBlockstore(dir) + await fs.open() + await fs.close() + })() + ).to.eventually.be.undefined() + }) + }) + + describe('open', () => { + it('createIfMissing: false - folder missing', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const store = new FsBlockstore(dir, { createIfMissing: false }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_OPEN_FAILED') + }) + + it('errorIfExists: true - folder exists', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await fs.mkdir(dir, { + recursive: true + }) + const store = new FsBlockstore(dir, { errorIfExists: true }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_OPEN_FAILED') + }) + }) + + it('deleting files', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + await fs.put(key, Uint8Array.from([0, 1, 2, 3])) + await fs.delete(key) + + await expect(fs.get(key)).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + + it('deleting non-existent files', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + + await fs.delete(key) + + await expect(fs.get(key)).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + + describe('interface-datastore (flat directory)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { + shardingStrategy: new FlatDirectory() + }) + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + describe('interface-datastore (default sharding)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`)) + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + describe('interface-datastore (custom encoding)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { + shardingStrategy: new NextToLast({ + base: base256emoji + }) + }) + + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + it('can survive concurrent writes', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const value = utf8Encoder.encode('Hello world') + + await Promise.all( + new Array(100).fill(0).map(async () => { await fs.put(key, value) }) + ) + + const res = await fs.get(key) + + expect(res).to.deep.equal(value) + }) +}) diff --git a/packages/blockstore-filestore/test/sharding.spec.ts b/packages/blockstore-filestore/test/sharding.spec.ts new file mode 100644 index 00000000..3c896570 --- /dev/null +++ b/packages/blockstore-filestore/test/sharding.spec.ts @@ -0,0 +1,109 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import { base32upper } from 'multiformats/bases/base32' +import { CID } from 'multiformats/cid' +import { FlatDirectory, NextToLast } from '../src/sharding.js' + +describe('flat', () => { + it('should encode', () => { + const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const strategy = new FlatDirectory() + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('') + expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.data`) + }) + + it('should encode with extension', () => { + const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const strategy = new FlatDirectory({ + extension: '.file' + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('') + expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.file`) + }) + + it('should decode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new FlatDirectory() + const cid = strategy.decode(`${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new FlatDirectory({ + extension: '.file' + }) + const cid = strategy.decode(`${mh}.file`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) +}) + +describe('next to last', () => { + it('should encode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast() + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('LY') + expect(file).to.equal(`${mh}.data`) + }) + + it('should encode with prefix length', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast({ + prefixLength: 4 + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('QYLY') + expect(file).to.equal(`${mh}.data`) + }) + + it('should encode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast({ + extension: '.file' + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('LY') + expect(file).to.equal(`${mh}.file`) + }) + + it('should decode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast() + const cid = strategy.decode(`LY/${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with prefix length', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast({ + prefixLength: 4 + }) + const cid = strategy.decode(`QYLY/${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast({ + extension: '.file' + }) + const cid = strategy.decode(`LY/${mh}.file`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) +}) diff --git a/packages/blockstore-filestore/tsconfig.json b/packages/blockstore-filestore/tsconfig.json new file mode 100644 index 00000000..e27c7fa6 --- /dev/null +++ b/packages/blockstore-filestore/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../blockstore-core" + }, + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} From 4c8959d73cab640485853c1c211843e850ea46ee Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 13:49:33 +1200 Subject: [PATCH 02/14] Update changelog, readme and package. --- packages/blockstore-filestore/CHANGELOG.md | 52 ---------------------- packages/blockstore-filestore/README.md | 8 ++-- packages/blockstore-filestore/package.json | 13 +++--- 3 files changed, 8 insertions(+), 65 deletions(-) diff --git a/packages/blockstore-filestore/CHANGELOG.md b/packages/blockstore-filestore/CHANGELOG.md index dd7434c3..e69de29b 100644 --- a/packages/blockstore-filestore/CHANGELOG.md +++ b/packages/blockstore-filestore/CHANGELOG.md @@ -1,52 +0,0 @@ -## [blockstore-fs-v1.1.3](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.2...blockstore-fs-v1.1.3) (2023-06-03) - - -### Documentation - -* fix capitalization of import ([#226](https://github.com/ipfs/js-stores/issues/226)) ([837221a](https://github.com/ipfs/js-stores/commit/837221aff3ef4d217063eb17953aff03764e7600)) - -## [blockstore-fs-v1.1.2](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.1...blockstore-fs-v1.1.2) (2023-06-03) - - -### Dependencies - -* bump aegir from 38.1.8 to 39.0.9 ([#225](https://github.com/ipfs/js-stores/issues/225)) ([d0f301b](https://github.com/ipfs/js-stores/commit/d0f301b1243a0f4f692011449567b51b2706e70f)) - -## [blockstore-fs-v1.1.1](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.1.0...blockstore-fs-v1.1.1) (2023-03-31) - - -### Trivial Changes - -* rename master to main ([#200](https://github.com/ipfs/js-stores/issues/200)) ([f85d719](https://github.com/ipfs/js-stores/commit/f85d719b711cd60237bdaa6a0bcd418e69a98598)) - - -### Dependencies - -* update all it-* deps ([#213](https://github.com/ipfs/js-stores/issues/213)) ([e963497](https://github.com/ipfs/js-stores/commit/e963497fdb33e61e2fe702866abbd42fba648fee)) - -## [blockstore-fs-v1.1.0](https://github.com/ipfs/js-stores/compare/blockstore-fs-v1.0.1...blockstore-fs-v1.1.0) (2023-03-23) - - -### Features - -* add all blockstore and datastore implementations ([#197](https://github.com/ipfs/js-stores/issues/197)) ([0d85128](https://github.com/ipfs/js-stores/commit/0d851286d48c357b07df3f7419c1e903ed0e7fac)) - -## [1.0.1](https://github.com/ipfs/js-blockstore-fs/compare/v1.0.0...v1.0.1) (2023-03-23) - - -### Dependencies - -* update interface-store to 5.x.x ([#1](https://github.com/ipfs/js-blockstore-fs/issues/1)) ([88fa91c](https://github.com/ipfs/js-blockstore-fs/commit/88fa91cb1405ed66f053ed265c1690ac0ad22214)) - -## 1.0.0 (2023-03-13) - - -### Bug Fixes - -* update ci build files ([bdc1881](https://github.com/ipfs/js-blockstore-fs/commit/bdc18810e6d63ffdbf6fc6617757aaa96b0ba82c)) - - -### Trivial Changes - -* add missing dep ([f5ba415](https://github.com/ipfs/js-blockstore-fs/commit/f5ba41536816f08eb1b97f298091c7221c6c9360)) -* initial import ([7a576cb](https://github.com/ipfs/js-blockstore-fs/commit/7a576cbad5696ad396c0cd2d557edf71d624a860)) diff --git a/packages/blockstore-filestore/README.md b/packages/blockstore-filestore/README.md index e46e8afa..1074c4c7 100644 --- a/packages/blockstore-filestore/README.md +++ b/packages/blockstore-filestore/README.md @@ -18,21 +18,19 @@ ## Install ```console -$ npm i blockstore-fs +$ npm i blockstore-filestore ``` ## Usage ```js -import { FsBlockstore } from 'blockstore-fs' +import { FilestoreBlockstore } from 'blockstore-filestore' -const store = new FsBlockstore('path/to/store') +const store = new FilestoreBlockstore(blockstore, datastore) ``` ## API Docs -- - ## License Licensed under either of diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json index a7942fb4..df77eef7 100644 --- a/packages/blockstore-filestore/package.json +++ b/packages/blockstore-filestore/package.json @@ -1,9 +1,9 @@ { - "name": "blockstore-fs", - "version": "1.1.3", - "description": "Blockstore implementation with file system backend", + "name": "blockstore-filestore", + "version": "0.1.0", + "description": "Blockstore implementation with no-copy file system backend", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-fs#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-filestore#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/js-stores.git" @@ -12,6 +12,7 @@ "url": "https://github.com/ipfs/js-stores/issues" }, "keywords": [ + "filestore", "datastore", "fs", "interface", @@ -50,10 +51,6 @@ ".": { "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" - }, - "./sharding": { - "types": "./dist/src/sharding.d.ts", - "import": "./dist/src/sharding.js" } }, "eslintConfig": { From fb9c8adc36768cff4c83ac53b1561c6b90ed1b6a Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 13:50:44 +1200 Subject: [PATCH 03/14] Remove blockstore-fs specific code. --- .../benchmarks/encoding/package.json | 18 -- .../benchmarks/encoding/src/README.md | 31 --- .../benchmarks/encoding/src/index.ts | 73 ----- .../benchmarks/encoding/tsconfig.json | 13 - packages/blockstore-filestore/src/index.ts | 254 ------------------ packages/blockstore-filestore/src/sharding.ts | 117 -------- .../blockstore-filestore/test/index.spec.ts | 161 ----------- .../test/sharding.spec.ts | 109 -------- 8 files changed, 776 deletions(-) delete mode 100644 packages/blockstore-filestore/benchmarks/encoding/package.json delete mode 100644 packages/blockstore-filestore/benchmarks/encoding/src/README.md delete mode 100644 packages/blockstore-filestore/benchmarks/encoding/src/index.ts delete mode 100644 packages/blockstore-filestore/benchmarks/encoding/tsconfig.json delete mode 100644 packages/blockstore-filestore/src/sharding.ts delete mode 100644 packages/blockstore-filestore/test/index.spec.ts delete mode 100644 packages/blockstore-filestore/test/sharding.spec.ts diff --git a/packages/blockstore-filestore/benchmarks/encoding/package.json b/packages/blockstore-filestore/benchmarks/encoding/package.json deleted file mode 100644 index 00104d4f..00000000 --- a/packages/blockstore-filestore/benchmarks/encoding/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "benchmarks-encoding", - "version": "1.0.0", - "main": "index.js", - "private": true, - "type": "module", - "scripts": { - "clean": "aegir clean", - "build": "aegir build --bundle false", - "lint": "aegir lint", - "dep-check": "aegir dep-check", - "start": "npm run build && node dist/src/index.js" - }, - "devDependencies": { - "multiformats": "^11.0.1", - "tinybench": "^2.4.0" - } -} diff --git a/packages/blockstore-filestore/benchmarks/encoding/src/README.md b/packages/blockstore-filestore/benchmarks/encoding/src/README.md deleted file mode 100644 index 3caf1e9e..00000000 --- a/packages/blockstore-filestore/benchmarks/encoding/src/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Encoding Benchmark - -Multiformats ships a number of base encoding algorithms. This module has no strong opinion -on which is best, as long as it is case insensitive, so benchmark them to choose the fastest. - -At the time of writing `base8` is the fastest, followed other alorithms using `rfc4648` encoding -internally in `multiformats` (e.g. `base16`, `base32`), and finally anything using `baseX` encoding. - -We choose base32upper which uses `rfc4648` because it has a longer alphabet so will shard better. - -## Usage - -```console -$ npm i -$ npm start - -> benchmarks-gc@1.0.0 start -> npm run build && node dist/src/index.js - - -> benchmarks-gc@1.0.0 build -> aegir build --bundle false - -[14:51:28] tsc [started] -[14:51:33] tsc [completed] -generating Ed25519 keypair... -┌─────────┬────────────────┬─────────┬───────────┬──────┐ -│ (index) │ Implementation │ ops/s │ ms/op │ runs │ -├─────────┼────────────────┼─────────┼───────────┼──────┤ -//... results here -``` diff --git a/packages/blockstore-filestore/benchmarks/encoding/src/index.ts b/packages/blockstore-filestore/benchmarks/encoding/src/index.ts deleted file mode 100644 index 40fc0905..00000000 --- a/packages/blockstore-filestore/benchmarks/encoding/src/index.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { base10 } from 'multiformats/bases/base10' -import { base16upper } from 'multiformats/bases/base16' -import { base256emoji } from 'multiformats/bases/base256emoji' -import { base32, base32upper, base32hexupper, base32z } from 'multiformats/bases/base32' -import { base36, base36upper } from 'multiformats/bases/base36' -import { base8 } from 'multiformats/bases/base8' -import { CID } from 'multiformats/cid' -import { Bench } from 'tinybench' - -const RESULT_PRECISION = 2 - -const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - -async function main (): Promise { - const suite = new Bench() - suite.add('base8', () => { - base8.encode(cid.bytes) - }) - suite.add('base10', () => { - base10.encode(cid.bytes) - }) - suite.add('base16upper', () => { - base16upper.encode(cid.bytes) - }) - suite.add('base32', () => { - base32.encode(cid.bytes) - }) - suite.add('base32upper', () => { - base32upper.encode(cid.bytes) - }) - suite.add('base32hexupper', () => { - base32hexupper.encode(cid.bytes) - }) - suite.add('base32z', () => { - base32z.encode(cid.bytes) - }) - suite.add('base36', () => { - base36.encode(cid.bytes) - }) - suite.add('base36upper', () => { - base36upper.encode(cid.bytes) - }) - suite.add('base256emoji', () => { - base256emoji.encode(cid.bytes) - }) - - await suite.run() - - console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console - const resultA = a.result?.hz ?? 0 - const resultB = b.result?.hz ?? 0 - - if (resultA === resultB) { - return 0 - } - - if (resultA < resultB) { - return 1 - } - - return -1 - }).map(({ name, result }) => ({ - Implementation: name, - 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), - 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), - runs: result?.samples.length - }))) -} - -main().catch(err => { - console.error(err) // eslint-disable-line no-console - process.exit(1) -}) diff --git a/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json b/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json deleted file mode 100644 index fee64009..00000000 --- a/packages/blockstore-filestore/benchmarks/encoding/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "aegir/src/config/tsconfig.aegir.json", - "compilerOptions": { - "outDir": "dist", - "target": "ES2022", - "module": "ES2022", - "lib": ["ES2022", "DOM", "DOM.Iterable"] - }, - "include": [ - "src", - "test" - ] -} diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index 06471475..e69de29b 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -1,254 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'node:path' -import { promisify } from 'node:util' -import { - Errors -} from 'blockstore-core' -// @ts-expect-error no types -import fwa from 'fast-write-atomic' -import glob from 'it-glob' -import map from 'it-map' -import parallelBatch from 'it-parallel-batch' -import { NextToLast, type ShardingStrategy } from './sharding.js' -import type { Blockstore, Pair } from 'interface-blockstore' -import type { AwaitIterable } from 'interface-store' -import type { CID } from 'multiformats/cid' - -const writeAtomic = promisify(fwa) - -/** - * Write a file atomically - */ -async function writeFile (file: string, contents: Uint8Array): Promise { - try { - await writeAtomic(file, contents) - } catch (err: any) { - if (err.code === 'EPERM' && err.syscall === 'rename') { - // fast-write-atomic writes a file to a temp location before renaming it. - // On Windows, if the final file already exists this error is thrown. - // No such error is thrown on Linux/Mac - // Make sure we can read & write to this file - await fs.access(file, fs.constants.F_OK | fs.constants.W_OK) - - // The file was created by another context - this means there were - // attempts to write the same block by two different function calls - return - } - - throw err - } -} - -export interface FsBlockstoreInit { - /** - * If true and the passed blockstore location does not exist, create - * it on startup. default: true - */ - createIfMissing?: boolean - - /** - * If true and the passed blockstore location exists on startup, throw - * an error. default: false - */ - errorIfExists?: boolean - - /** - * The file extension to use when storing blocks. default: '.data' - */ - extension?: string - - /** - * How many blocks to put in parallel when `.putMany` is called. - * default: 50 - */ - putManyConcurrency?: number - - /** - * How many blocks to read in parallel when `.getMany` is called. - * default: 50 - */ - getManyConcurrency?: number - - /** - * How many blocks to delete in parallel when `.deleteMany` is called. - * default: 50 - */ - deleteManyConcurrency?: number - - /** - * Control how CIDs map to paths and back - */ - shardingStrategy?: ShardingStrategy -} - -/** - * A blockstore backed by the file system - */ -export class FsBlockstore implements Blockstore { - public path: string - private readonly createIfMissing: boolean - private readonly errorIfExists: boolean - private readonly putManyConcurrency: number - private readonly getManyConcurrency: number - private readonly deleteManyConcurrency: number - private readonly shardingStrategy: ShardingStrategy - - constructor (location: string, init: FsBlockstoreInit = {}) { - this.path = path.resolve(location) - this.createIfMissing = init.createIfMissing ?? true - this.errorIfExists = init.errorIfExists ?? false - this.deleteManyConcurrency = init.deleteManyConcurrency ?? 50 - this.getManyConcurrency = init.getManyConcurrency ?? 50 - this.putManyConcurrency = init.putManyConcurrency ?? 50 - this.shardingStrategy = init.shardingStrategy ?? new NextToLast() - } - - async open (): Promise { - try { - await fs.access(this.path, fs.constants.F_OK | fs.constants.W_OK) - - if (this.errorIfExists) { - throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} already exists`)) - } - } catch (err: any) { - if (err.code === 'ENOENT') { - if (this.createIfMissing) { - await fs.mkdir(this.path, { recursive: true }) - return - } else { - throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} does not exist`)) - } - } - - throw err - } - } - - async close (): Promise { - await Promise.resolve() - } - - async put (key: CID, val: Uint8Array): Promise { - const { dir, file } = this.shardingStrategy.encode(key) - - try { - if (dir != null && dir !== '') { - await fs.mkdir(path.join(this.path, dir), { - recursive: true - }) - } - - await writeFile(path.join(this.path, dir, file), val) - - return key - } catch (err: any) { - throw Errors.putFailedError(err) - } - } - - async * putMany (source: AwaitIterable): AsyncIterable { - yield * parallelBatch( - map(source, ({ cid, block }) => { - return async () => { - await this.put(cid, block) - - return cid - } - }), - this.putManyConcurrency - ) - } - - async get (key: CID): Promise { - const { dir, file } = this.shardingStrategy.encode(key) - - try { - return await fs.readFile(path.join(this.path, dir, file)) - } catch (err: any) { - throw Errors.notFoundError(err) - } - } - - async * getMany (source: AwaitIterable): AsyncIterable { - yield * parallelBatch( - map(source, key => { - return async () => { - return { - cid: key, - block: await this.get(key) - } - } - }), - this.getManyConcurrency - ) - } - - async delete (key: CID): Promise { - const { dir, file } = this.shardingStrategy.encode(key) - - try { - await fs.unlink(path.join(this.path, dir, file)) - } catch (err: any) { - if (err.code === 'ENOENT') { - return - } - - throw Errors.deleteFailedError(err) - } - } - - async * deleteMany (source: AwaitIterable): AsyncIterable { - yield * parallelBatch( - map(source, key => { - return async () => { - await this.delete(key) - - return key - } - }), - this.deleteManyConcurrency - ) - } - - /** - * Check for the existence of the given key - */ - async has (key: CID): Promise { - const { dir, file } = this.shardingStrategy.encode(key) - - try { - await fs.access(path.join(this.path, dir, file)) - } catch (err: any) { - return false - } - return true - } - - async * getAll (): AsyncIterable { - const pattern = `**/*${this.shardingStrategy.extension}` - .split(path.sep) - .join('/') - const files = glob(this.path, pattern, { - absolute: true - }) - - for await (const file of files) { - try { - const buf = await fs.readFile(file) - - const pair: Pair = { - cid: this.shardingStrategy.decode(file), - block: buf - } - - yield pair - } catch (err: any) { - // if keys are removed from the datastore while the query is - // running, we may encounter missing files. - if (err.code !== 'ENOENT') { - throw err - } - } - } - } -} diff --git a/packages/blockstore-filestore/src/sharding.ts b/packages/blockstore-filestore/src/sharding.ts deleted file mode 100644 index 10d65169..00000000 --- a/packages/blockstore-filestore/src/sharding.ts +++ /dev/null @@ -1,117 +0,0 @@ -import path from 'node:path' -import { base32upper } from 'multiformats/bases/base32' -import { CID } from 'multiformats/cid' -import type { MultibaseCodec } from 'multiformats/bases/interface' - -export interface ShardingStrategy { - extension: string - encode: (cid: CID) => { dir: string, file: string } - decode: (path: string) => CID -} - -export interface NextToLastInit { - /** - * The file extension to use. default: '.data' - */ - extension?: string - - /** - * How many characters to take from the end of the CID. default: 2 - */ - prefixLength?: number - - /** - * The multibase codec to use - nb. should be case insensitive. - * default: base32upper - */ - base?: MultibaseCodec -} - -/** - * A sharding strategy that takes the last few characters of a multibase encoded - * CID and uses them as the directory to store the block in. This prevents - * storing all blocks in a single directory which would overwhelm most - * filesystems. - */ -export class NextToLast implements ShardingStrategy { - public extension: string - private readonly prefixLength: number - private readonly base: MultibaseCodec - - constructor (init: NextToLastInit = {}) { - this.extension = init.extension ?? '.data' - this.prefixLength = init.prefixLength ?? 2 - this.base = init.base ?? base32upper - } - - encode (cid: CID): { dir: string, file: string } { - const str = this.base.encoder.encode(cid.multihash.bytes) - const prefix = str.substring(str.length - this.prefixLength) - - return { - dir: prefix, - file: `${str}${this.extension}` - } - } - - decode (str: string): CID { - let fileName = path.basename(str) - - if (fileName.endsWith(this.extension)) { - fileName = fileName.substring(0, fileName.length - this.extension.length) - } - - return CID.decode(this.base.decoder.decode(fileName)) - } -} - -export interface FlatDirectoryInit { - /** - * The file extension to use. default: '.data' - */ - extension?: string - - /** - * How many characters to take from the end of the CID. default: 2 - */ - prefixLength?: number - - /** - * The multibase codec to use - nb. should be case insensitive. - * default: base32padupper - */ - base?: MultibaseCodec -} - -/** - * A sharding strategy that does not do any sharding and stores all files - * in one directory. Only for testing, do not use in production. - */ -export class FlatDirectory implements ShardingStrategy { - public extension: string - private readonly base: MultibaseCodec - - constructor (init: NextToLastInit = {}) { - this.extension = init.extension ?? '.data' - this.base = init.base ?? base32upper - } - - encode (cid: CID): { dir: string, file: string } { - const str = this.base.encoder.encode(cid.multihash.bytes) - - return { - dir: '', - file: `${str}${this.extension}` - } - } - - decode (str: string): CID { - let fileName = path.basename(str) - - if (fileName.endsWith(this.extension)) { - fileName = fileName.substring(0, fileName.length - this.extension.length) - } - - return CID.decode(this.base.decoder.decode(fileName)) - } -} diff --git a/packages/blockstore-filestore/test/index.spec.ts b/packages/blockstore-filestore/test/index.spec.ts deleted file mode 100644 index 25e0ebc3..00000000 --- a/packages/blockstore-filestore/test/index.spec.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* eslint-env mocha */ -import fs from 'node:fs/promises' -import os from 'node:os' -import path from 'node:path' -import { expect } from 'aegir/chai' -import { interfaceBlockstoreTests } from 'interface-blockstore-tests' -import { base256emoji } from 'multiformats/bases/base256emoji' -import { CID } from 'multiformats/cid' -import { FsBlockstore } from '../src/index.js' -import { FlatDirectory, NextToLast } from '../src/sharding.js' - -const utf8Encoder = new TextEncoder() - -describe('FsBlockstore', () => { - describe('construction', () => { - it('defaults - folder missing', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - await expect( - (async () => { - const fs = new FsBlockstore(dir) - await fs.open() - await fs.close() - })() - ).to.eventually.be.undefined() - }) - - it('defaults - folder exists', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - await fs.mkdir(dir, { - recursive: true - }) - await expect( - (async () => { - const fs = new FsBlockstore(dir) - await fs.open() - await fs.close() - })() - ).to.eventually.be.undefined() - }) - }) - - describe('open', () => { - it('createIfMissing: false - folder missing', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - const store = new FsBlockstore(dir, { createIfMissing: false }) - await expect(store.open()).to.eventually.be.rejected - .with.property('code', 'ERR_OPEN_FAILED') - }) - - it('errorIfExists: true - folder exists', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - await fs.mkdir(dir, { - recursive: true - }) - const store = new FsBlockstore(dir, { errorIfExists: true }) - await expect(store.open()).to.eventually.be.rejected - .with.property('code', 'ERR_OPEN_FAILED') - }) - }) - - it('deleting files', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - const fs = new FsBlockstore(dir) - await fs.open() - - const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - await fs.put(key, Uint8Array.from([0, 1, 2, 3])) - await fs.delete(key) - - await expect(fs.get(key)).to.eventually.be.rejected - .with.property('code', 'ERR_NOT_FOUND') - }) - - it('deleting non-existent files', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - const fs = new FsBlockstore(dir) - await fs.open() - - const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - - await fs.delete(key) - - await expect(fs.get(key)).to.eventually.be.rejected - .with.property('code', 'ERR_NOT_FOUND') - }) - - describe('interface-datastore (flat directory)', () => { - interfaceBlockstoreTests({ - setup: async () => { - const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { - shardingStrategy: new FlatDirectory() - }) - await store.open() - - return store - }, - teardown: async (store) => { - await store.close() - await fs.rm(store.path, { - recursive: true - }) - } - }) - }) - - describe('interface-datastore (default sharding)', () => { - interfaceBlockstoreTests({ - setup: async () => { - const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`)) - await store.open() - - return store - }, - teardown: async (store) => { - await store.close() - await fs.rm(store.path, { - recursive: true - }) - } - }) - }) - - describe('interface-datastore (custom encoding)', () => { - interfaceBlockstoreTests({ - setup: async () => { - const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { - shardingStrategy: new NextToLast({ - base: base256emoji - }) - }) - - await store.open() - - return store - }, - teardown: async (store) => { - await store.close() - await fs.rm(store.path, { - recursive: true - }) - } - }) - }) - - it('can survive concurrent writes', async () => { - const dir = path.join(os.tmpdir(), `test-${Math.random()}`) - const fs = new FsBlockstore(dir) - await fs.open() - - const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - const value = utf8Encoder.encode('Hello world') - - await Promise.all( - new Array(100).fill(0).map(async () => { await fs.put(key, value) }) - ) - - const res = await fs.get(key) - - expect(res).to.deep.equal(value) - }) -}) diff --git a/packages/blockstore-filestore/test/sharding.spec.ts b/packages/blockstore-filestore/test/sharding.spec.ts deleted file mode 100644 index 3c896570..00000000 --- a/packages/blockstore-filestore/test/sharding.spec.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* eslint-env mocha */ -import { expect } from 'aegir/chai' -import { base32upper } from 'multiformats/bases/base32' -import { CID } from 'multiformats/cid' -import { FlatDirectory, NextToLast } from '../src/sharding.js' - -describe('flat', () => { - it('should encode', () => { - const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - const strategy = new FlatDirectory() - const { dir, file } = strategy.encode(cid) - - expect(dir).to.equal('') - expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.data`) - }) - - it('should encode with extension', () => { - const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') - const strategy = new FlatDirectory({ - extension: '.file' - }) - const { dir, file } = strategy.encode(cid) - - expect(dir).to.equal('') - expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.file`) - }) - - it('should decode', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const strategy = new FlatDirectory() - const cid = strategy.decode(`${mh}.data`) - - expect(cid).to.eql(CID.decode(base32upper.decode(mh))) - }) - - it('should decode with extension', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const strategy = new FlatDirectory({ - extension: '.file' - }) - const cid = strategy.decode(`${mh}.file`) - - expect(cid).to.eql(CID.decode(base32upper.decode(mh))) - }) -}) - -describe('next to last', () => { - it('should encode', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const cid = CID.decode(base32upper.decode(mh)) - const strategy = new NextToLast() - const { dir, file } = strategy.encode(cid) - - expect(dir).to.equal('LY') - expect(file).to.equal(`${mh}.data`) - }) - - it('should encode with prefix length', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const cid = CID.decode(base32upper.decode(mh)) - const strategy = new NextToLast({ - prefixLength: 4 - }) - const { dir, file } = strategy.encode(cid) - - expect(dir).to.equal('QYLY') - expect(file).to.equal(`${mh}.data`) - }) - - it('should encode with extension', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const cid = CID.decode(base32upper.decode(mh)) - const strategy = new NextToLast({ - extension: '.file' - }) - const { dir, file } = strategy.encode(cid) - - expect(dir).to.equal('LY') - expect(file).to.equal(`${mh}.file`) - }) - - it('should decode', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const strategy = new NextToLast() - const cid = strategy.decode(`LY/${mh}.data`) - - expect(cid).to.eql(CID.decode(base32upper.decode(mh))) - }) - - it('should decode with prefix length', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const strategy = new NextToLast({ - prefixLength: 4 - }) - const cid = strategy.decode(`QYLY/${mh}.data`) - - expect(cid).to.eql(CID.decode(base32upper.decode(mh))) - }) - - it('should decode with extension', () => { - const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' - const strategy = new NextToLast({ - extension: '.file' - }) - const cid = strategy.decode(`LY/${mh}.file`) - - expect(cid).to.eql(CID.decode(base32upper.decode(mh))) - }) -}) From 70e46d2277e35166e23ccd2246ca265c3bf0ef86 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 13:53:44 +1200 Subject: [PATCH 04/14] Add dependencies. --- packages/blockstore-filestore/package.json | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json index df77eef7..2f58465a 100644 --- a/packages/blockstore-filestore/package.json +++ b/packages/blockstore-filestore/package.json @@ -160,18 +160,15 @@ "docs": "aegir docs" }, "dependencies": { - "blockstore-core": "^4.0.0", - "fast-write-atomic": "^0.2.0", - "interface-blockstore": "^5.0.0", - "interface-store": "^5.0.0", - "it-glob": "^2.0.1", - "it-map": "^3.0.1", - "it-parallel-batch": "^3.0.0", - "multiformats": "^11.0.2" + "interface-datastore": "^8.2.3", + "multiformats": "^12.0.1", + "protons-runtime": "^5.0.0" }, "devDependencies": { "aegir": "^39.0.9", - "interface-blockstore-tests": "^6.0.0" + "interface-blockstore": "^5.2.3", + "interface-store": "^5.1.2", + "protons": "^7.0.2" }, "typedoc": { "entryPoint": "./src/index.ts" From c99bc09cb4958c099ad61f4215205db8f8aa3290 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 14:24:05 +1200 Subject: [PATCH 05/14] Add protobuf files. --- packages/blockstore-filestore/package.json | 1 + .../blockstore-filestore/src/pb/dataobj.proto | 7 ++ .../blockstore-filestore/src/pb/dataobj.ts | 87 +++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 packages/blockstore-filestore/src/pb/dataobj.proto create mode 100644 packages/blockstore-filestore/src/pb/dataobj.ts diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json index 2f58465a..d9097842 100644 --- a/packages/blockstore-filestore/package.json +++ b/packages/blockstore-filestore/package.json @@ -152,6 +152,7 @@ "clean": "aegir clean", "lint": "aegir lint", "build": "aegir build --bundle false", + "generate": "protons ./src/pb/dataobj.proto", "release": "aegir release", "test": "aegir test -t node -t electron-main", "test:node": "aegir test -t node", diff --git a/packages/blockstore-filestore/src/pb/dataobj.proto b/packages/blockstore-filestore/src/pb/dataobj.proto new file mode 100644 index 00000000..2bb35772 --- /dev/null +++ b/packages/blockstore-filestore/src/pb/dataobj.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +message DataObj { + string FilePath = 1; + uint64 Offset = 2; + uint32 Size = 3; +} diff --git a/packages/blockstore-filestore/src/pb/dataobj.ts b/packages/blockstore-filestore/src/pb/dataobj.ts new file mode 100644 index 00000000..08586a65 --- /dev/null +++ b/packages/blockstore-filestore/src/pb/dataobj.ts @@ -0,0 +1,87 @@ +/* eslint-disable import/export */ +/* eslint-disable complexity */ +/* eslint-disable @typescript-eslint/no-namespace */ +/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ +/* eslint-disable @typescript-eslint/no-empty-interface */ + +import { encodeMessage, decodeMessage, message } from 'protons-runtime' +import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' + +export interface DataObj { + FilePath: string + Offset: bigint + Size: number +} + +export namespace DataObj { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.FilePath != null && obj.FilePath !== '')) { + w.uint32(10) + w.string(obj.FilePath) + } + + if ((obj.Offset != null && obj.Offset !== 0n)) { + w.uint32(16) + w.uint64(obj.Offset) + } + + if ((obj.Size != null && obj.Size !== 0)) { + w.uint32(24) + w.uint32(obj.Size) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + FilePath: '', + Offset: 0n, + Size: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: + obj.FilePath = reader.string() + break + case 2: + obj.Offset = reader.uint64() + break + case 3: + obj.Size = reader.uint32() + break + default: + reader.skipType(tag & 7) + break + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, DataObj.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): DataObj => { + return decodeMessage(buf, DataObj.codec()) + } +} From a08e09471aa12db8a2b78636b2f7ef7d0d0300de Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 14:24:13 +1200 Subject: [PATCH 06/14] Add utils. --- packages/blockstore-filestore/src/utils.ts | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/blockstore-filestore/src/utils.ts diff --git a/packages/blockstore-filestore/src/utils.ts b/packages/blockstore-filestore/src/utils.ts new file mode 100644 index 00000000..9bcefef4 --- /dev/null +++ b/packages/blockstore-filestore/src/utils.ts @@ -0,0 +1,48 @@ +import fs from 'node:fs' +import { Key } from 'interface-datastore' +import { base32 } from 'multiformats/bases/base32' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import * as Digest from 'multiformats/hashes/digest' + +export const cidToKey = (cid: CID): Key => { + return new Key(`/${base32.encode(cid.multihash.bytes).slice(1).toUpperCase()}`, false) +} + +export const keyToCid = (key: Key): CID => { + return CID.createV1(raw.code, Digest.decode(base32.decode(`b${key.toString().slice(1).toLowerCase()}`))) +} + +export const readChunk = async (path: string, offset: bigint, size: number): Promise => { + const fd = await new Promise((resolve, reject) => { + fs.open(path, (err, fd) => { + if (err != null) { + reject(err) + } else { + resolve(fd) + } + }) + }) + + const data = await new Promise((resolve, reject) => { + fs.read(fd, { position: offset, length: size }, (err, _, data: Uint8Array) => { + if (err != null) { + reject(err) + } else { + resolve(data) + } + }) + }) + + await new Promise((resolve, reject) => { + fs.close(fd, (err) => { + if (err != null) { + reject(err) + } else { + resolve() + } + }) + }) + + return data +} From 9cdbcd0772f5c20d8e61a80347afa96018f7cc89 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 30 Jun 2023 14:26:36 +1200 Subject: [PATCH 07/14] Add main filestore code. --- packages/blockstore-filestore/README.md | 4 +- packages/blockstore-filestore/src/index.ts | 108 +++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/packages/blockstore-filestore/README.md b/packages/blockstore-filestore/README.md index 1074c4c7..860ccb4d 100644 --- a/packages/blockstore-filestore/README.md +++ b/packages/blockstore-filestore/README.md @@ -24,9 +24,9 @@ $ npm i blockstore-filestore ## Usage ```js -import { FilestoreBlockstore } from 'blockstore-filestore' +import { Filestore } from 'blockstore-filestore' -const store = new FilestoreBlockstore(blockstore, datastore) +const store = new Filestore(blockstore, datastore) ``` ## API Docs diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index e69de29b..6f5ad8a5 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -0,0 +1,108 @@ +import { DataObj } from './pb/dataobj.js' +import { readChunk, cidToKey, keyToCid } from './utils.js' +import type { Blockstore, Pair } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' +import type { AbortOptions, AwaitIterable, Await } from 'interface-store' +import type { CID } from 'multiformats/cid' + +export class Filestore implements Blockstore { + private readonly blockstore: Blockstore + private readonly datastore: Datastore + + constructor (blockstore: Blockstore, datastore: Datastore) { + this.blockstore = blockstore + this.datastore = datastore + } + + async get (key: CID, options?: AbortOptions): Promise { + if (await this.blockstore.has(key, options)) { + const block = await this.blockstore.get(key, options) + + return block + } + + const dKey = cidToKey(key) + const index = await this.datastore.get(dKey, options) + const dataObj = DataObj.decode(index) + const chunk = await readChunk(dataObj.FilePath, dataObj.Offset, dataObj.Size) + + return chunk + } + + async * getMany (source: AwaitIterable, options?: AbortOptions): AsyncGenerator { + for await (const cid of source) { + const block = await this.get(cid, options) + + yield { cid, block } + } + } + + async * getAll (options?: AbortOptions): AsyncGenerator { + yield * this.blockstore.getAll(options) + + const keys = this.datastore.queryKeys({ filters: [() => true] }, options) + + for await (const key of keys) { + let cid: CID + + try { + cid = keyToCid(key) + } catch (error) { + continue + } + + const block = await this.get(cid, options) + + yield { block, cid } + } + } + + async has (key: CID): Promise { + if (await this.blockstore.has(key)) { + return true + } + + const dKey = cidToKey(key) + const hasKey = await this.datastore.has(dKey) + + return hasKey + } + + put (key: CID, val: Uint8Array, options?: AbortOptions): Await { + return this.blockstore.put(key, val, options) + } + + putMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { + return this.blockstore.putMany(source, options) + } + + async delete (key: CID, options?: AbortOptions): Promise { + const dKey = cidToKey(key) + + await Promise.all([ + this.blockstore.delete(key, options), + this.datastore.delete(dKey, options) + ]) + } + + async * deleteMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { + for await (const cid of source) { + await this.delete(cid, options) + yield cid + } + } + + async putLink (key: CID, path: string, offset: bigint, size: number, options?: AbortOptions): Promise { + const data = DataObj.encode({ + FilePath: path, + Offset: offset, + Size: size + }) + + const dKey = cidToKey(key) + + await this.datastore.put(dKey, data, options) + + return key + } +} From c7b72b0a3af2905022eacf9b910c8ba0146bb0eb Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 3 Jul 2023 13:59:53 +1200 Subject: [PATCH 08/14] Make size a bigint. --- packages/blockstore-filestore/src/index.ts | 2 +- packages/blockstore-filestore/src/pb/dataobj.proto | 2 +- packages/blockstore-filestore/src/pb/dataobj.ts | 10 +++++----- packages/blockstore-filestore/src/utils.ts | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index 6f5ad8a5..6e5b1624 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -92,7 +92,7 @@ export class Filestore implements Blockstore { } } - async putLink (key: CID, path: string, offset: bigint, size: number, options?: AbortOptions): Promise { + async putLink (key: CID, path: string, offset: bigint, size: bigint, options?: AbortOptions): Promise { const data = DataObj.encode({ FilePath: path, Offset: offset, diff --git a/packages/blockstore-filestore/src/pb/dataobj.proto b/packages/blockstore-filestore/src/pb/dataobj.proto index 2bb35772..d60990a8 100644 --- a/packages/blockstore-filestore/src/pb/dataobj.proto +++ b/packages/blockstore-filestore/src/pb/dataobj.proto @@ -3,5 +3,5 @@ syntax = "proto3"; message DataObj { string FilePath = 1; uint64 Offset = 2; - uint32 Size = 3; + uint64 Size = 3; } diff --git a/packages/blockstore-filestore/src/pb/dataobj.ts b/packages/blockstore-filestore/src/pb/dataobj.ts index 08586a65..a540b21b 100644 --- a/packages/blockstore-filestore/src/pb/dataobj.ts +++ b/packages/blockstore-filestore/src/pb/dataobj.ts @@ -11,7 +11,7 @@ import type { Uint8ArrayList } from 'uint8arraylist' export interface DataObj { FilePath: string Offset: bigint - Size: number + Size: bigint } export namespace DataObj { @@ -34,9 +34,9 @@ export namespace DataObj { w.uint64(obj.Offset) } - if ((obj.Size != null && obj.Size !== 0)) { + if ((obj.Size != null && obj.Size !== 0n)) { w.uint32(24) - w.uint32(obj.Size) + w.uint64(obj.Size) } if (opts.lengthDelimited !== false) { @@ -46,7 +46,7 @@ export namespace DataObj { const obj: any = { FilePath: '', Offset: 0n, - Size: 0 + Size: 0n } const end = length == null ? reader.len : reader.pos + length @@ -62,7 +62,7 @@ export namespace DataObj { obj.Offset = reader.uint64() break case 3: - obj.Size = reader.uint32() + obj.Size = reader.uint64() break default: reader.skipType(tag & 7) diff --git a/packages/blockstore-filestore/src/utils.ts b/packages/blockstore-filestore/src/utils.ts index 9bcefef4..cc9ce28d 100644 --- a/packages/blockstore-filestore/src/utils.ts +++ b/packages/blockstore-filestore/src/utils.ts @@ -13,7 +13,7 @@ export const keyToCid = (key: Key): CID => { return CID.createV1(raw.code, Digest.decode(base32.decode(`b${key.toString().slice(1).toLowerCase()}`))) } -export const readChunk = async (path: string, offset: bigint, size: number): Promise => { +export const readChunk = async (path: string, offset: bigint, size: bigint): Promise => { const fd = await new Promise((resolve, reject) => { fs.open(path, (err, fd) => { if (err != null) { @@ -25,7 +25,7 @@ export const readChunk = async (path: string, offset: bigint, size: number): Pro }) const data = await new Promise((resolve, reject) => { - fs.read(fd, { position: offset, length: size }, (err, _, data: Uint8Array) => { + fs.read(fd, { position: offset, length: Number(size) }, (err, _, data: Uint8Array) => { if (err != null) { reject(err) } else { From b56adce8e946cb5a3d6958863b2d41bbe5e91756 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 3 Jul 2023 15:09:19 +1200 Subject: [PATCH 09/14] Fix buffer overflow. --- packages/blockstore-filestore/src/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/blockstore-filestore/src/utils.ts b/packages/blockstore-filestore/src/utils.ts index cc9ce28d..c5ff75be 100644 --- a/packages/blockstore-filestore/src/utils.ts +++ b/packages/blockstore-filestore/src/utils.ts @@ -25,7 +25,11 @@ export const readChunk = async (path: string, offset: bigint, size: bigint): Pro }) const data = await new Promise((resolve, reject) => { - fs.read(fd, { position: offset, length: Number(size) }, (err, _, data: Uint8Array) => { + fs.read(fd, { + position: offset, + length: Number(size), + buffer: Buffer.alloc(Number(size)) + }, (err, _, data: Uint8Array) => { if (err != null) { reject(err) } else { From 8d198d10512100e064598427a4979bd22ddfc7a6 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 3 Jul 2023 15:58:37 +1200 Subject: [PATCH 10/14] Format chunk as node. --- packages/blockstore-filestore/package.json | 1 + packages/blockstore-filestore/src/index.ts | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json index d9097842..0b81fbd5 100644 --- a/packages/blockstore-filestore/package.json +++ b/packages/blockstore-filestore/package.json @@ -161,6 +161,7 @@ "docs": "aegir docs" }, "dependencies": { + "@ipld/dag-pb": "^4.0.4", "interface-datastore": "^8.2.3", "multiformats": "^12.0.1", "protons-runtime": "^5.0.0" diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index 6e5b1624..b88a9abd 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -1,5 +1,6 @@ import { DataObj } from './pb/dataobj.js' import { readChunk, cidToKey, keyToCid } from './utils.js' +import * as dagPb from '@ipld/dag-pb' import type { Blockstore, Pair } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { AbortOptions, AwaitIterable, Await } from 'interface-store' @@ -26,7 +27,12 @@ export class Filestore implements Blockstore { const dataObj = DataObj.decode(index) const chunk = await readChunk(dataObj.FilePath, dataObj.Offset, dataObj.Size) - return chunk + const block = dagPb.encode({ + Links: [], + Data: chunk + }); + + return block } async * getMany (source: AwaitIterable, options?: AbortOptions): AsyncGenerator { From ac091901ab7be79a3d537bbafc5890e8d6a05121 Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 4 Jul 2023 11:28:14 +1200 Subject: [PATCH 11/14] Return raw chunks instead of files. --- packages/blockstore-filestore/package.json | 1 - packages/blockstore-filestore/src/index.ts | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/blockstore-filestore/package.json b/packages/blockstore-filestore/package.json index 0b81fbd5..d9097842 100644 --- a/packages/blockstore-filestore/package.json +++ b/packages/blockstore-filestore/package.json @@ -161,7 +161,6 @@ "docs": "aegir docs" }, "dependencies": { - "@ipld/dag-pb": "^4.0.4", "interface-datastore": "^8.2.3", "multiformats": "^12.0.1", "protons-runtime": "^5.0.0" diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index b88a9abd..e9531a70 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -1,10 +1,10 @@ import { DataObj } from './pb/dataobj.js' import { readChunk, cidToKey, keyToCid } from './utils.js' -import * as dagPb from '@ipld/dag-pb' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' import type { Blockstore, Pair } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { AbortOptions, AwaitIterable, Await } from 'interface-store' -import type { CID } from 'multiformats/cid' export class Filestore implements Blockstore { private readonly blockstore: Blockstore @@ -26,13 +26,14 @@ export class Filestore implements Blockstore { const index = await this.datastore.get(dKey, options) const dataObj = DataObj.decode(index) const chunk = await readChunk(dataObj.FilePath, dataObj.Offset, dataObj.Size) + const hash = await sha256.digest(chunk) + const cid = CID.create(1, 0x55, hash) - const block = dagPb.encode({ - Links: [], - Data: chunk - }); + if (!cid.equals(key)) { + throw new Error("CID does not match.") + } - return block + return chunk } async * getMany (source: AwaitIterable, options?: AbortOptions): AsyncGenerator { From b0fb4227f6734a30f2f6346de156d8b2905854cf Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 4 Jul 2023 12:56:50 +1200 Subject: [PATCH 12/14] Cleanup CID creation. --- packages/blockstore-filestore/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index e9531a70..bef1e65f 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -2,6 +2,7 @@ import { DataObj } from './pb/dataobj.js' import { readChunk, cidToKey, keyToCid } from './utils.js' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' +import * as raw from 'multiformats/codecs/raw' import type { Blockstore, Pair } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { AbortOptions, AwaitIterable, Await } from 'interface-store' @@ -27,7 +28,7 @@ export class Filestore implements Blockstore { const dataObj = DataObj.decode(index) const chunk = await readChunk(dataObj.FilePath, dataObj.Offset, dataObj.Size) const hash = await sha256.digest(chunk) - const cid = CID.create(1, 0x55, hash) + const cid = CID.createV1(raw.code, hash) if (!cid.equals(key)) { throw new Error("CID does not match.") From bbc1bc5ad9df2cea50f303c08d2f959a3ab2ff2d Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 4 Jul 2023 12:59:06 +1200 Subject: [PATCH 13/14] Linting. --- packages/blockstore-filestore/src/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index bef1e65f..d4de6edb 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -1,8 +1,8 @@ -import { DataObj } from './pb/dataobj.js' -import { readChunk, cidToKey, keyToCid } from './utils.js' import { CID } from 'multiformats/cid' -import { sha256 } from 'multiformats/hashes/sha2' import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import { DataObj } from './pb/dataobj.js' +import { readChunk, cidToKey, keyToCid } from './utils.js' import type { Blockstore, Pair } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { AbortOptions, AwaitIterable, Await } from 'interface-store' @@ -31,7 +31,7 @@ export class Filestore implements Blockstore { const cid = CID.createV1(raw.code, hash) if (!cid.equals(key)) { - throw new Error("CID does not match.") + throw new Error('CID does not match.') } return chunk From 717cc8ff21407aeb9e44ab63eaac370123a0c9bb Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 4 Jul 2023 13:42:14 +1200 Subject: [PATCH 14/14] Use CID version based of key. --- packages/blockstore-filestore/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/blockstore-filestore/src/index.ts b/packages/blockstore-filestore/src/index.ts index d4de6edb..8b544c73 100644 --- a/packages/blockstore-filestore/src/index.ts +++ b/packages/blockstore-filestore/src/index.ts @@ -28,7 +28,7 @@ export class Filestore implements Blockstore { const dataObj = DataObj.decode(index) const chunk = await readChunk(dataObj.FilePath, dataObj.Offset, dataObj.Size) const hash = await sha256.digest(chunk) - const cid = CID.createV1(raw.code, hash) + const cid = CID.create(key.version, raw.code, hash) if (!cid.equals(key)) { throw new Error('CID does not match.')