From 691ade4c0707aa3e4492c6216f96283ad659fd53 Mon Sep 17 00:00:00 2001 From: Alexander Marks Date: Wed, 1 Sep 2021 11:54:44 -0700 Subject: [PATCH 1/3] Add new lit-dev-tools-esm package The lit-dev-tools package is currently CommonJS, because mostly it is used for Eleventy plugins, and Eleventy doesn't support ES modules (https://github.com/11ty/eleventy/issues/836). We want ES modules for this new redirect checker script, because it needs to import some ES modules, and that is difficult to do with TypeScript, because TypeScript doesn't allow emitting an actual `import` statement, which is how CommonJS -> ESM interop works (https://github.com/microsoft/TypeScript/issues/43329). We also an't really have a mix of CommonJS and ESM in the same package, because the {"type": "module"} field has to be set. We could use .mjs extensions, but TypeScript won't emit those. So the simplest solution seems to be to just have two packages! --- packages/lit-dev-tools-esm/package.json | 16 ++++++++++++++++ packages/lit-dev-tools-esm/tsconfig.json | 22 ++++++++++++++++++++++ packages/lit-dev-tools/package.json | 2 +- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 packages/lit-dev-tools-esm/package.json create mode 100644 packages/lit-dev-tools-esm/tsconfig.json diff --git a/packages/lit-dev-tools-esm/package.json b/packages/lit-dev-tools-esm/package.json new file mode 100644 index 000000000..2ab2209ed --- /dev/null +++ b/packages/lit-dev-tools-esm/package.json @@ -0,0 +1,16 @@ +{ + "name": "lit-dev-tools-esm", + "private": true, + "version": "0.0.0", + "description": "Misc tools for lit.dev (ES Modules)", + "author": "Google LLC", + "license": "BSD-3-Clause", + "type": "module", + "scripts": { + "build": "npm run build:ts", + "build:ts": "../../node_modules/.bin/tsc", + "format": "../../node_modules/.bin/prettier \"**/*.{ts,js,json,html,css,md}\" --write" + }, + "dependencies": { + } +} diff --git a/packages/lit-dev-tools-esm/tsconfig.json b/packages/lit-dev-tools-esm/tsconfig.json new file mode 100644 index 000000000..428eb597d --- /dev/null +++ b/packages/lit-dev-tools-esm/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "esnext", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./lib", + "rootDir": "./src", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": [] +} diff --git a/packages/lit-dev-tools/package.json b/packages/lit-dev-tools/package.json index f4f50baa9..297ec0e4d 100644 --- a/packages/lit-dev-tools/package.json +++ b/packages/lit-dev-tools/package.json @@ -2,7 +2,7 @@ "name": "lit-dev-tools", "private": true, "version": "0.0.0", - "description": "Misc tools for lit.dev", + "description": "Misc tools for lit.dev (CommonJS)", "author": "Google LLC", "license": "BSD-3-Clause", "scripts": { From 24e8ba232ccc24db3e3f9af593720991d64b6d37 Mon Sep 17 00:00:00 2001 From: Alexander Marks Date: Wed, 1 Sep 2021 10:48:02 -0700 Subject: [PATCH 2/3] Add custom script for checking lit.dev redirects --- .github/workflows/build.yml | 3 + package.json | 3 +- packages/lit-dev-tools-esm/package-lock.json | 142 ++++++++++++++++++ packages/lit-dev-tools-esm/package.json | 4 + .../lit-dev-tools-esm/src/check-redirects.ts | 124 +++++++++++++++ packages/lit-dev-tools/package.json | 4 +- 6 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 packages/lit-dev-tools-esm/package-lock.json create mode 100644 packages/lit-dev-tools-esm/src/check-redirects.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e4ce7de9..1555fa43c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,9 @@ jobs: - name: Build run: npm run build + - name: Check for broken redirects + run: npm run test:links:redirects + - name: Check for broken links (internal) run: npm run test:links:internal diff --git a/package.json b/package.json index 1e93fcfb2..b9c7952d2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "format": "prettier \"**/*.{ts,js,json,html,css,md}\" --write", "nuke": "rm -rf node_modules package-lock.json packages/*/node_modules packages/*/package-lock.json && npm install && npx lerna bootstrap", "test": "npm run test:links", - "test:links": "npm run test:links:internal && npm run test:links:external", + "test:links": "npm run test:links:redirects && npm run test:links:internal && npm run test:links:external", + "test:links:redirects": "node packages/lit-dev-tools-esm/lib/check-redirects.js", "test:links:internal": "run-p -r start check-links:internal", "test:links:external": "run-p -r start check-links:external", "check-links:internal": "wait-on tcp:8080 && blc http://localhost:8080 --recursive --exclude-external --ordered", diff --git a/packages/lit-dev-tools-esm/package-lock.json b/packages/lit-dev-tools-esm/package-lock.json new file mode 100644 index 000000000..f33037e4f --- /dev/null +++ b/packages/lit-dev-tools-esm/package-lock.json @@ -0,0 +1,142 @@ +{ + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "license": "BSD-3-Clause", + "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "node-fetch": "^3.0.0" + } + }, + "node_modules/@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "node_modules/ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "dependencies": { + "array-back": "^6.2.0" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "dependencies": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==", + "engines": { + "node": ">= 8" + } + } + }, + "dependencies": { + "@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "requires": { + "array-back": "^6.2.0" + } + }, + "array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==" + }, + "data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==" + }, + "fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "requires": { + "web-streams-polyfill": "^3.0.3" + } + }, + "node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "requires": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + } + }, + "web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==" + } + } +} diff --git a/packages/lit-dev-tools-esm/package.json b/packages/lit-dev-tools-esm/package.json index 2ab2209ed..9b5a1336e 100644 --- a/packages/lit-dev-tools-esm/package.json +++ b/packages/lit-dev-tools-esm/package.json @@ -12,5 +12,9 @@ "format": "../../node_modules/.bin/prettier \"**/*.{ts,js,json,html,css,md}\" --write" }, "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "lit-dev-server": "^0.0.0", + "node-fetch": "^3.0.0" } } diff --git a/packages/lit-dev-tools-esm/src/check-redirects.ts b/packages/lit-dev-tools-esm/src/check-redirects.ts new file mode 100644 index 000000000..3aa47e85f --- /dev/null +++ b/packages/lit-dev-tools-esm/src/check-redirects.ts @@ -0,0 +1,124 @@ +/** + * @license + * Copyright 2021 Google LLC + * SPDX-License-Identifier: BSD-3-Clause + */ + +import * as pathLib from 'path'; +import * as fs from 'fs/promises'; +import ansi from 'ansi-escape-sequences'; +import fetch from 'node-fetch'; +import {pageRedirects} from 'lit-dev-server/redirects.js'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = pathLib.dirname(__filename); + +const {red, green, yellow, bold, reset} = ansi.style; + +const OK = Symbol(); +type ErrorMessage = string; + +const isUrl = (str: string) => { + try { + new URL(str); + return true; + } catch { + return false; + } +}; + +const trimTrailingSlash = (str: string) => + str.endsWith('/') ? str.slice(0, str.length - 1) : str; + +const siteOutputDir = pathLib.resolve( + __dirname, + '../', + '../', + 'lit-dev-content', + '_site' +); + +const checkRedirect = async ( + redirect: string +): Promise => { + if (isUrl(redirect)) { + // Remote URLs. + let res; + try { + res = await fetch(redirect); + } catch (e) { + return `Fetch error: ${(e as Error).message}`; + } + if (res.status !== 200) { + return `HTTP ${res.status} error`; + } + } else { + // Local paths. A bit hacky, but since we know how Eleventy works, we don't + // need to actually run the server, we can just look directly in the built + // HTML output directory. + const {pathname, hash} = new URL(redirect, 'http://lit.dev'); + const diskPath = pathLib.relative( + process.cwd(), + pathLib.join(siteOutputDir, trimTrailingSlash(pathname), 'index.html') + ); + let data; + try { + data = await fs.readFile(diskPath, {encoding: 'utf8'}); + } catch { + return `Could not find file matching path ${pathname} +Searched for file ${diskPath}`; + } + if (hash) { + // Another hack. Just do a regexp search for e.g. id="somesection" instead + // of DOM parsing. Should be good enough, especially given how regular our + // Markdown generated HTML is. + const idAttrRegExp = new RegExp(`\\sid=["']?${hash.slice(1)}["']?[\\s>]`); + if (data.match(idAttrRegExp) === null) { + return `Could not find section matching hash ${hash}. +Searched in file ${diskPath}`; + } + } + } + return OK; +}; + +const checkAllRedirects = async () => { + console.log('=========================='); + console.log('Checking lit.dev redirects'); + console.log('=========================='); + console.log(); + + let fail = false; + const promises = []; + for (const [from, to] of pageRedirects) { + promises.push( + (async () => { + const result = await checkRedirect(to); + if (result === OK) { + console.log(`${bold + green}OK${reset} ${from} -> ${to}`); + } else { + console.log(); + console.log( + `${bold + red}BROKEN REDIRECT${reset} ${from} -> ${ + yellow + to + reset + }` + ); + console.log(result); + console.log(); + fail = true; + } + })() + ); + } + await Promise.all(promises); + console.log(); + if (fail) { + console.log('Redirects were broken!'); + process.exit(1); + } else { + console.error('All redirects OK!'); + } +}; + +checkAllRedirects(); diff --git a/packages/lit-dev-tools/package.json b/packages/lit-dev-tools/package.json index 297ec0e4d..d7fe912a4 100644 --- a/packages/lit-dev-tools/package.json +++ b/packages/lit-dev-tools/package.json @@ -12,8 +12,10 @@ "test": "npm run build && uvu ./lib \".spec.js$\"" }, "dependencies": { + "@types/jsdom": "^16.2.13", "@types/markdown-it": "^12.0.1", "@types/source-map": "^0.5.7", + "@types/strip-comments": "^2.0.1", "@web/dev-server": "^0.1.6", "@web/dev-server-core": "^0.3.5", "fast-glob": "^3.2.5", @@ -27,8 +29,6 @@ "strip-comments": "^2.0.1", "striptags": "^3.2.0", "typedoc": "^0.20.30", - "@types/jsdom": "^16.2.13", - "@types/strip-comments": "^2.0.1", "uvu": "^0.5.1" } } From e20dfb12f313e109cd3b7aabde88ef887bd7507a Mon Sep 17 00:00:00 2001 From: Alexander Marks Date: Wed, 1 Sep 2021 12:36:14 -0700 Subject: [PATCH 3/3] Clarify isUrl -> isAbsoluteUrl --- packages/lit-dev-tools-esm/src/check-redirects.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lit-dev-tools-esm/src/check-redirects.ts b/packages/lit-dev-tools-esm/src/check-redirects.ts index 3aa47e85f..b67eddfdf 100644 --- a/packages/lit-dev-tools-esm/src/check-redirects.ts +++ b/packages/lit-dev-tools-esm/src/check-redirects.ts @@ -19,7 +19,7 @@ const {red, green, yellow, bold, reset} = ansi.style; const OK = Symbol(); type ErrorMessage = string; -const isUrl = (str: string) => { +const isAbsoluteUrl = (str: string) => { try { new URL(str); return true; @@ -42,7 +42,7 @@ const siteOutputDir = pathLib.resolve( const checkRedirect = async ( redirect: string ): Promise => { - if (isUrl(redirect)) { + if (isAbsoluteUrl(redirect)) { // Remote URLs. let res; try {