From d1edb2da0e9ce855095ebd4661d9f9f99375ac66 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 12 Jun 2024 22:34:08 -0500 Subject: [PATCH] chore: Cache workspace dependency builds (#715) --- .gitignore | 1 + package.json | 9 ++- packages/wxt-demo/package.json | 16 ++-- pnpm-lock.yaml | 42 ++++++++++- scripts/build-deps.ts | 133 +++++++++++++++++++++++++++++++++ 5 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 scripts/build-deps.ts diff --git a/.gitignore b/.gitignore index 31fbfa9f..60fe67a3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ templates/*/package-lock.json docs/api/reference stats.html .tool-versions +.cache diff --git a/package.json b/package.json index 1e5a6695..ad600833 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "docs:gen": "typedoc --options docs/typedoc.json", "docs:dev": "pnpm -s docs:gen && vitepress dev docs", "docs:build": "pnpm -s docs:gen && vitepress build docs", - "docs:preview": "pnpm -s docs:gen && vitepress preview docs" + "docs:preview": "pnpm -s docs:gen && vitepress preview docs", + "build-deps": "pnpm tsx scripts/build-deps.ts" }, "devDependencies": { "@aklinker1/check": "^1.1.1", @@ -23,8 +24,11 @@ "@vitest/coverage-v8": "^1.0.1", "changelogen": "^0.5.5", "consola": "^3.2.3", + "dependency-graph": "^1.0.0", "execa": "^9.1.0", + "fast-glob": "^3.3.1", "fs-extra": "^11.1.1", + "hasha": "^6.0.0", "lint-staged": "^15.2.0", "npm-run-all": "^4.1.5", "prettier": "^3.3.0", @@ -39,7 +43,8 @@ "vitest-mock-extended": "^1.3.1", "vitest-plugin-random-seed": "^1.0.2", "vue": "^3.3.10", - "wxt": "workspace:*" + "wxt": "workspace:*", + "yaml": "^2.4.5" }, "simple-git-hooks": { "pre-commit": "pnpm lint-staged" diff --git a/packages/wxt-demo/package.json b/packages/wxt-demo/package.json index 9dcd5765..628db51d 100644 --- a/packages/wxt-demo/package.json +++ b/packages/wxt-demo/package.json @@ -4,18 +4,18 @@ "private": true, "type": "module", "scripts": { - "dev": "pnpm build:deps && wxt", - "build:deps": "pnpm --filter wxt build", - "build": "pnpm build:deps && wxt build", - "build:all": "pnpm build:deps && run-s -s 'build:all:*'", + "dev": "pnpm -s build-deps && wxt", + "build": "pnpm -s build-deps && wxt build", + "build:all": "pnpm -s build-deps && run-s -s 'build:all:*'", "build:all:chrome-mv3": "wxt build", "build:all:chrome-mv2": "wxt build --mv2", "build:all:firefox-mv3": "wxt build -b firefox --mv3", "build:all:firefox-mv2": "wxt build -b firefox", - "test": "pnpm build:deps && vitest", - "zip": "pnpm build:deps && wxt zip", - "check": "pnpm build:deps && check", - "postinstall": "pnpm build:deps && wxt prepare" + "test": "pnpm -s build-deps && vitest", + "zip": "pnpm -s build-deps && wxt zip", + "check": "pnpm -s build-deps && check", + "postinstall": "pnpm -s build-deps && wxt prepare", + "build-deps": "pnpm -sw build-deps wxt-demo" }, "dependencies": { "react": "^18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9afdda53..5e10cb86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,12 +23,21 @@ importers: consola: specifier: ^3.2.3 version: 3.2.3 + dependency-graph: + specifier: ^1.0.0 + version: 1.0.0 execa: specifier: ^9.1.0 version: 9.1.0 + fast-glob: + specifier: ^3.3.1 + version: 3.3.2 fs-extra: specifier: ^11.1.1 version: 11.2.0 + hasha: + specifier: ^6.0.0 + version: 6.0.0 lint-staged: specifier: ^15.2.0 version: 15.2.2 @@ -74,6 +83,11 @@ importers: wxt: specifier: workspace:* version: link:packages/wxt + yaml: + specifier: ^2.4.5 + version: 2.4.5 + + packages/storage: {} packages/wxt: dependencies: @@ -2218,7 +2232,7 @@ packages: scule: 1.3.0 semver: 7.5.4 std-env: 3.6.0 - yaml: 2.3.4 + yaml: 2.4.5 dev: true /check-error@1.0.3: @@ -2578,6 +2592,11 @@ packages: /defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + /dependency-graph@1.0.0: + resolution: {integrity: sha512-cW3gggJ28HZ/LExwxP2B++aiKxhJXMSIt9K48FOXQkm+vuG5gyatXnLsONRJdzO/7VfjDIiaOOa/bs4l464Lwg==} + engines: {node: '>=4'} + dev: true + /dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -3338,6 +3357,14 @@ packages: function-bind: 1.1.1 dev: true + /hasha@6.0.0: + resolution: {integrity: sha512-MLydoyGp9QJcjlhE5lsLHXYpWayjjWqkavzju2ZWD2tYa1CgmML1K1gWAu22BLFa2eZ0OfvJ/DlfoVjaD54U2Q==} + engines: {node: '>=18'} + dependencies: + is-stream: 3.0.0 + type-fest: 4.20.0 + dev: true + /highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} dev: false @@ -4635,7 +4662,7 @@ packages: optional: true dependencies: lilconfig: 2.1.0 - yaml: 2.3.4 + yaml: 2.4.5 dev: true /postcss@8.4.38: @@ -5558,6 +5585,11 @@ packages: resolution: {integrity: sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==} engines: {node: '>=14.16'} + /type-fest@4.20.0: + resolution: {integrity: sha512-MBh+PHUHHisjXf4tlx0CFWoMdjx8zCMLJHOjnV1prABYZFHqtFOyauCIK2/7w4oIfwkF8iNhLtnJEfVY2vn3iw==} + engines: {node: '>=16'} + dev: true + /typed-array-length@1.0.4: resolution: {integrity: sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==} dependencies: @@ -6233,6 +6265,12 @@ packages: engines: {node: '>= 14'} dev: true + /yaml@2.4.5: + resolution: {integrity: sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==} + engines: {node: '>= 14'} + hasBin: true + dev: true + /yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} diff --git a/scripts/build-deps.ts b/scripts/build-deps.ts new file mode 100644 index 00000000..dc90222f --- /dev/null +++ b/scripts/build-deps.ts @@ -0,0 +1,133 @@ +// +// Build all workspace packages, or only dependencies of a specific package. +// Kind of a simple version of nx/turborepo, but we can keep using PNPM to run +// all commands. +// +// Usage: +// pnpm tsx scripts/build-deps.ts # Builds all packages in repo +// pnpm tsx scripts/build-deps.ts # Builds only dependencies of +// + +import glob from 'fast-glob'; +import { DepGraph } from 'dependency-graph'; +import fs from 'fs-extra'; +import YAML from 'yaml'; +import { execa } from 'execa'; +import consola from 'consola'; +import { hashFile, hash } from 'hasha'; +import { resolve } from 'node:path'; + +// Quick hack to prevent recursive calls to this script. If it's being called +// from another instance of this script, the parent instance is already +// preforming this build, so it can be skipped. +if (process.env.BUILD_DEPS) process.exit(0); +process.env.BUILD_DEPS = 'true'; + +// Parse args +const packageToBuild = process.argv[2]; +if (packageToBuild == null) { + consola.info('Building all packages...'); +} else { + consola.info(`Building dependencies of \`${packageToBuild}\``); +} + +// Build graph +const workspace: { packages: string[] } = YAML.parse( + await fs.readFile('pnpm-workspace.yaml', 'utf8'), +); +const packageDirs = await glob(workspace.packages, { onlyDirectories: true }); + +const graph = new DepGraph>(); +// Add all packages to chart +for (const packageDir of packageDirs) { + try { + const packageJson = await fs.readJson(`${packageDir}/package.json`); + graph.addNode(packageJson.name, { ...packageJson, dir: packageDir }); + } catch { + // Package missing package.json, ignore it + } +} +// Add dependencies between packages +graph.entryNodes().forEach((packageName) => { + const packageJson = graph.getNodeData(packageName); + [ + ...Object.entries(packageJson.dependencies ?? {}), + ...Object.entries(packageJson.devDependencies ?? {}), + ].forEach(([dep, version]) => { + if (version !== 'workspace:*') return; + graph.addDependency(packageName, dep); + }); +}); +function printGraph(graph: DepGraph) { + consola.debug('Dependency Graph:'); + function printNode(node: string, level = 0) { + consola.debug(`${''.padStart(level * 2, ' ')}- \`${node}\``); + graph.dependenciesOf(node).forEach((dep) => printNode(dep, level + 1)); + } + graph.entryNodes().forEach((entry) => printNode(entry)); +} +printGraph(graph); +if (packageToBuild) { + // Remove nodes not associated with the package to build + graph.entryNodes().forEach((entry) => { + if (entry !== packageToBuild) graph.removeNode(entry); + }); +} + +// Get order +const buildOrder = graph.overallOrder().filter( + // Remove packageToBuild if provided + (packageName) => packageName !== packageToBuild, +); +consola.info('Build order:', buildOrder); + +// Build dependencies of a package +async function hashDir(dir: string): Promise { + const files = await glob('**/*', { + ignore: [ + '**/dist/**', + '**/node_modules/**', + '**/__tests__/**', + '**/e2e/**', + 'CHANGELOG.md', + 'typedoc.json', + 'vitest.*.ts', + ], + cwd: dir, + }); + const hashes = await Promise.all( + files.sort().map(async (file) => { + const hash = await hashFile(resolve(dir, file), { algorithm: 'md5' }); + return `${hash}-${file}`; + }), + ); + consola.debug(hashes.join('\n')); + return await hash(hashes.join('\n'), { algorithm: 'md5' }); +} +async function buildPackage(packageName: string): Promise { + const packageJson = graph.getNodeData(packageName); + const hash = await hashDir(packageJson.dir); + consola.debug('Directory hash:', hash); + const cacheDir = resolve('.cache', packageJson.dir, hash); + const outputDir = resolve(packageJson.dir, 'dist'); + if (await fs.pathExists(cacheDir)) { + await fs.ensureDir(outputDir); + await fs.copy(cacheDir, outputDir); + consola.success(`\`${packageName}\` cached`); + } else { + await execa('pnpm', ['--filter', packageName, 'build'], { + stdio: 'inherit', + }); + await fs.ensureDir(cacheDir); + await fs.copy(outputDir, cacheDir); + } +} + +try { + for (const packageName of buildOrder) { + await buildPackage(packageName); + } +} catch (err) { + consola.error(err); + process.exit(1); +}