diff --git a/.gitattributes b/.gitattributes index e03e86ab..485306a9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ * text=auto eol=lf pnpm-lock.yaml linguist-generated +package-lock.json linguist-generated +bun.lockb linguist-generated +yarn.lock linguist-generated diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 53555ca6..76e27811 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -43,6 +43,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup + - uses: oven-sh/setup-bun@v1 - name: pnpm test:coverage run: pnpm test:coverage --reporter=default --reporter=hanging-process - uses: codecov/codecov-action@v4 diff --git a/demo/wxt.config.ts b/demo/wxt.config.ts index 8df3dc92..c3201f52 100644 --- a/demo/wxt.config.ts +++ b/demo/wxt.config.ts @@ -15,4 +15,7 @@ export default defineConfig({ alias: { public: 'src/public', }, + zip: { + downloadPackages: ['sass'], + }, }); diff --git a/docs/guide/publishing.md b/docs/guide/publishing.md index 332ceb9e..b609ceb2 100644 --- a/docs/guide/publishing.md +++ b/docs/guide/publishing.md @@ -136,6 +136,28 @@ See Issue [#377](https://github.com/wxt-dev/wxt/issues/377) for more details. Ensure that you have a `README.md` or `SOURCE_CODE_REVIEW.md` file with the above commands so that the Firefox team knows how to build your extension. +#### Private Packages + +If you use private packages and you don't want to provide your auth token to the Firefox team during the review process, you can use `zip.downloadPackages` to download any private packages and include them in the zip. + +```ts +// wxt.config.ts +export default defineConfig({ + zip: { + downloadPackages: [ + '@mycompany/some-package', + //... + ], + }, +}); +``` + +Depending on your package manager, the `package.json` in the sources zip will be modified to use the downloaded dependencies via the `overrides` or `resolutions` field. + +:::warning +WXT uses the command `npm pack ` to download the package. That means regardless of your package manager, you need to properly setup a `.npmrc` file. NPM and PNPM both respect `.npmrc` files, but Yarn and Bun have their own ways of authorizing private registries, so you'll need to add an `.npmrc` file. +::: + ### Safari > 🚧 Not supported yet diff --git a/e2e/tests/zip.test.ts b/e2e/tests/zip.test.ts new file mode 100644 index 00000000..aa44fc30 --- /dev/null +++ b/e2e/tests/zip.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { TestProject } from '../utils'; +import extract from 'extract-zip'; +import { execaCommand } from 'execa'; +import { readFile, writeFile } from 'fs-extra'; + +process.env.WXT_PNPM_IGNORE_WORKSPACE = 'true'; + +describe('Zipping', () => { + it('should download packages and produce a valid build when zipping sources', async () => { + const project = new TestProject({ + name: 'test', + version: '1.0.0', + dependencies: { + flatten: '1.0.3', + }, + }); + project.addFile( + 'entrypoints/background.ts', + 'export default defineBackground(() => {});', + ); + const unzipDir = project.resolvePath('.output/test-1.0.0-sources'); + const sourcesZip = project.resolvePath('.output/test-1.0.0-sources.zip'); + + await project.zip({ + browser: 'firefox', + zip: { downloadPackages: ['flatten'] }, + }); + expect(await project.fileExists('.output/')).toBe(true); + + await extract(sourcesZip, { dir: unzipDir }); + // Update package json wxt path + const packageJsonPath = project.resolvePath(unzipDir, 'package.json'); + const packageJson = JSON.parse(await readFile(packageJsonPath, 'utf-8')); + packageJson.dependencies.wxt = '../../../../..'; + await writeFile( + packageJsonPath, + JSON.stringify(packageJson, null, 2), + 'utf-8', + ); + + // Build zipped extension + await expect( + execaCommand('pnpm i --ignore-workspace --frozen-lockfile false', { + cwd: unzipDir, + }), + ).resolves.toMatchObject({ exitCode: 0 }); + await expect( + execaCommand('pnpm wxt build -b firefox', { cwd: unzipDir }), + ).resolves.toMatchObject({ exitCode: 0 }); + + await expect(project.fileExists(unzipDir, '.output')).resolves.toBe(true); + expect( + await project.serializeFile( + project.resolvePath(unzipDir, 'package.json'), + ), + ).toMatchInlineSnapshot(` + ".output/test-1.0.0-sources/package.json + ---------------------------------------- + { + "name": "test", + "description": "Example description", + "version": "1.0.0", + "dependencies": { + "wxt": "../../../../..", + "flatten": "1.0.3" + }, + "resolutions": { + "flatten@1.0.3": "file://./.wxt/local_modules/flatten-1.0.3.tgz" + } + }" + `); + }); +}); diff --git a/e2e/utils.ts b/e2e/utils.ts index 84952ea3..3a0900f5 100644 --- a/e2e/utils.ts +++ b/e2e/utils.ts @@ -175,8 +175,8 @@ export class TestProject { ].join(`\n${''.padEnd(40, '-')}\n`); } - fileExists(path: string): Promise { - return fs.exists(this.resolvePath(path)); + fileExists(...path: string[]): Promise { + return fs.exists(this.resolvePath(...path)); } async getOutputManifest( diff --git a/package.json b/package.json index 47b495d8..f8f3907c 100644 --- a/package.json +++ b/package.json @@ -153,6 +153,7 @@ "@types/prompts": "^2.4.9", "@vitest/coverage-v8": "^1.0.1", "execa": "^8.0.1", + "extract-zip": "^2.0.1", "happy-dom": "^13.3.8", "lint-staged": "^15.2.0", "lodash.merge": "^4.6.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea63cecb..8c1729d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ importers: execa: specifier: ^8.0.1 version: 8.0.1 + extract-zip: + specifier: ^2.0.1 + version: 2.0.1 happy-dom: specifier: ^13.3.8 version: 13.3.8 @@ -1180,7 +1183,6 @@ packages: requiresBuild: true dependencies: '@types/node': 20.10.3 - dev: false optional: true /@vitejs/plugin-vue@5.0.2(vite@5.1.3)(vue@3.4.3): @@ -1715,7 +1717,6 @@ packages: /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} - dev: false /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -2269,7 +2270,6 @@ packages: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} dependencies: once: 1.4.0 - dev: false /entities@4.5.0: resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} @@ -2491,7 +2491,6 @@ packages: '@types/yauzl': 2.10.3 transitivePeerDependencies: - supports-color - dev: false /fast-glob@3.3.2: resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} @@ -2512,7 +2511,6 @@ packages: resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} dependencies: pend: 1.2.0 - dev: false /filesize@10.1.0: resolution: {integrity: sha512-GTLKYyBSDz3nPhlLVPjPWZCnhkd9TrrRArNcy8Z+J2cqScB7h2McAzR6NBX6nYOoWafql0roY8hrocxnZBv9CQ==} @@ -2669,7 +2667,6 @@ packages: engines: {node: '>=8'} dependencies: pump: 3.0.0 - dev: false /get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} @@ -4081,7 +4078,6 @@ packages: /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} - dev: false /perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} @@ -4236,7 +4232,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: false /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} @@ -5714,7 +5709,6 @@ packages: dependencies: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 - dev: false /yocto-queue@1.0.0: resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} diff --git a/src/core/package-managers/__tests__/bun.test.ts b/src/core/package-managers/__tests__/bun.test.ts new file mode 100644 index 00000000..4baa8cc9 --- /dev/null +++ b/src/core/package-managers/__tests__/bun.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { bun } from '../bun'; + +const cwd = path.resolve(__dirname, 'fixtures/bun-project'); + +describe.skipIf(() => process.platform === 'win32')( + 'Bun Package Management Utils', + () => { + describe('listDependencies', () => { + it('should list direct dependencies', async () => { + const actual = await bun.listDependencies({ cwd }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + + it('should list all dependencies', async () => { + const actual = await bun.listDependencies({ cwd, all: true }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-db', version: '1.52.0' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + }); + }, +); diff --git a/src/core/package-managers/__tests__/fixtures/bun-project/bun.lockb b/src/core/package-managers/__tests__/fixtures/bun-project/bun.lockb new file mode 100755 index 00000000..81553490 Binary files /dev/null and b/src/core/package-managers/__tests__/fixtures/bun-project/bun.lockb differ diff --git a/src/core/package-managers/__tests__/fixtures/bun-project/package.json b/src/core/package-managers/__tests__/fixtures/bun-project/package.json new file mode 100644 index 00000000..27ce0fce --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/bun-project/package.json @@ -0,0 +1,9 @@ +{ + "name": "bun-ls", + "dependencies": { + "mime-types": "2.1.35" + }, + "devDependencies": { + "flatten": "1.0.3" + } +} diff --git a/src/core/package-managers/__tests__/fixtures/npm-project/package-lock.json b/src/core/package-managers/__tests__/fixtures/npm-project/package-lock.json new file mode 100644 index 00000000..38503891 --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/npm-project/package-lock.json @@ -0,0 +1,42 @@ +{ + "name": "npm-ls", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "npm-ls", + "dependencies": { + "mime-types": "2.1.35" + }, + "devDependencies": { + "flatten": "1.0.3" + } + }, + "node_modules/flatten": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/flatten/-/flatten-1.0.3.tgz", + "integrity": "sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==", + "deprecated": "flatten is deprecated in favor of utility frameworks such as lodash.", + "dev": true + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + } + } +} diff --git a/src/core/package-managers/__tests__/fixtures/npm-project/package.json b/src/core/package-managers/__tests__/fixtures/npm-project/package.json new file mode 100644 index 00000000..8d06b7da --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/npm-project/package.json @@ -0,0 +1,9 @@ +{ + "name": "npm-ls", + "dependencies": { + "mime-types": "2.1.35" + }, + "devDependencies": { + "flatten": "1.0.3" + } +} diff --git a/src/core/package-managers/__tests__/fixtures/pnpm-project/package.json b/src/core/package-managers/__tests__/fixtures/pnpm-project/package.json new file mode 100644 index 00000000..f671946f --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/pnpm-project/package.json @@ -0,0 +1,9 @@ +{ + "name": "pnpm-ls", + "dependencies": { + "mime-types": "2.1.35" + }, + "devDependencies": { + "flatten": "1.0.3" + } +} diff --git a/src/core/package-managers/__tests__/fixtures/pnpm-project/pnpm-lock.yaml b/src/core/package-managers/__tests__/fixtures/pnpm-project/pnpm-lock.yaml new file mode 100644 index 00000000..0a0f7e1a --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/pnpm-project/pnpm-lock.yaml @@ -0,0 +1,34 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + mime-types: + specifier: 2.1.35 + version: 2.1.35 + +devDependencies: + flatten: + specifier: 1.0.3 + version: 1.0.3 + +packages: + + /flatten@1.0.3: + resolution: {integrity: sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==} + deprecated: flatten is deprecated in favor of utility frameworks such as lodash. + dev: true + + /mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: false + + /mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false diff --git a/src/core/package-managers/__tests__/fixtures/yarn-project/package.json b/src/core/package-managers/__tests__/fixtures/yarn-project/package.json new file mode 100644 index 00000000..b75f3418 --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/yarn-project/package.json @@ -0,0 +1,10 @@ +{ + "name": "yarn-ls", + "packageManager": "yarn@1.22.19", + "dependencies": { + "mime-types": "2.1.35" + }, + "devDependencies": { + "flatten": "1.0.3" + } +} diff --git a/src/core/package-managers/__tests__/fixtures/yarn-project/yarn.lock b/src/core/package-managers/__tests__/fixtures/yarn-project/yarn.lock new file mode 100644 index 00000000..8415c03c --- /dev/null +++ b/src/core/package-managers/__tests__/fixtures/yarn-project/yarn.lock @@ -0,0 +1,20 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +flatten@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b" + integrity sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg== + +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@2.1.35: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" diff --git a/src/core/package-managers/__tests__/npm.test.ts b/src/core/package-managers/__tests__/npm.test.ts new file mode 100644 index 00000000..ec36e15d --- /dev/null +++ b/src/core/package-managers/__tests__/npm.test.ts @@ -0,0 +1,46 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { npm } from '../npm'; +import { execaCommand } from 'execa'; +import { exists } from 'fs-extra'; + +const cwd = path.resolve(__dirname, 'fixtures/npm-project'); + +describe('NPM Package Management Utils', () => { + beforeAll(async () => { + // NPM needs the modules installed for 'npm ls' to work + await execaCommand('npm i', { cwd }); + }); + + describe('listDependencies', () => { + it('should list direct dependencies', async () => { + const actual = await npm.listDependencies({ cwd }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + + it('should list all dependencies', async () => { + const actual = await npm.listDependencies({ cwd, all: true }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + { name: 'mime-db', version: '1.52.0' }, + ]); + }); + }); + + describe('downloadDependency', () => { + it('should download the dependency as a tarball', async () => { + const downloadDir = path.resolve(cwd, 'dist'); + const id = 'mime-db@1.52.0'; + const expected = path.resolve(downloadDir, 'mime-db-1.52.0.tgz'); + + const actual = await npm.downloadDependency(id, downloadDir); + + expect(actual).toEqual(expected); + expect(await exists(actual)).toBe(true); + }); + }); +}); diff --git a/src/core/package-managers/__tests__/pnpm.test.ts b/src/core/package-managers/__tests__/pnpm.test.ts new file mode 100644 index 00000000..84edcb4c --- /dev/null +++ b/src/core/package-managers/__tests__/pnpm.test.ts @@ -0,0 +1,34 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { pnpm } from '../pnpm'; +import { execaCommand } from 'execa'; + +const cwd = path.resolve(__dirname, 'fixtures/pnpm-project'); + +process.env.WXT_PNPM_IGNORE_WORKSPACE = 'true'; + +describe('PNPM Package Management Utils', () => { + beforeAll(async () => { + // PNPM needs the modules installed, or 'pnpm ls' will return a blank list. + await execaCommand('pnpm i --ignore-workspace', { cwd }); + }); + + describe('listDependencies', () => { + it('should list direct dependencies', async () => { + const actual = await pnpm.listDependencies({ cwd }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + + it('should list all dependencies', async () => { + const actual = await pnpm.listDependencies({ cwd, all: true }); + expect(actual).toEqual([ + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + { name: 'mime-db', version: '1.52.0' }, + ]); + }); + }); +}); diff --git a/src/core/package-managers/__tests__/yarn.test.ts b/src/core/package-managers/__tests__/yarn.test.ts new file mode 100644 index 00000000..7195e04d --- /dev/null +++ b/src/core/package-managers/__tests__/yarn.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { yarn } from '../yarn'; + +const cwd = path.resolve(__dirname, 'fixtures/yarn-project'); + +describe('Yarn Package Management Utils', () => { + describe('listDependencies', () => { + it('should list direct dependencies', async () => { + const actual = await yarn.listDependencies({ cwd }); + expect(actual).toEqual([ + { name: 'mime-db', version: '1.52.0' }, + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + + it('should list all dependencies', async () => { + const actual = await yarn.listDependencies({ cwd, all: true }); + expect(actual).toEqual([ + { name: 'mime-db', version: '1.52.0' }, + { name: 'flatten', version: '1.0.3' }, + { name: 'mime-types', version: '2.1.35' }, + ]); + }); + }); +}); diff --git a/src/core/package-managers/bun.ts b/src/core/package-managers/bun.ts new file mode 100644 index 00000000..25b888b3 --- /dev/null +++ b/src/core/package-managers/bun.ts @@ -0,0 +1,27 @@ +import { dedupeDependencies, npm } from './npm'; +import { WxtPackageManagerImpl } from './types'; + +export const bun: WxtPackageManagerImpl = { + overridesKey: 'overrides', // But also supports "resolutions" + downloadDependency(...args) { + return npm.downloadDependency(...args); + }, + async listDependencies(options) { + const args = ['pm', 'ls']; + if (options?.all) { + args.push('--all'); + } + const { execa } = await import('execa'); + const res = await execa('bun', args, { cwd: options?.cwd }); + return dedupeDependencies( + res.stdout + .split('\n') + .slice(1) // Skip the first line, is not a dependency + .map((line) => line.trim()) + .map((line) => /.* (@?\S+)@(\S+)$/.exec(line)) + // @ts-expect-error: Filtering to known non-null matches + .filter((match) => !!match) + .map(([_, name, version]) => ({ name, version })), + ); + }, +}; diff --git a/src/core/package-managers/index.ts b/src/core/package-managers/index.ts index 3cee564f..ae2a25d3 100644 --- a/src/core/package-managers/index.ts +++ b/src/core/package-managers/index.ts @@ -6,8 +6,14 @@ import { installDependencies, removeDependency, PackageManager, + PackageManagerName, } from 'nypm'; import { WxtPackageManager } from '~/types'; +import { bun } from './bun'; +import { WxtPackageManagerImpl } from './types'; +import { yarn } from './yarn'; +import { pnpm } from './pnpm'; +import { npm } from './npm'; export async function createWxtPackageManager( root: string, @@ -46,5 +52,25 @@ export async function createWxtPackageManager( ensureDependencyInstalled, installDependencies, removeDependency, + get overridesKey() { + return requirePm((pm) => packageManagers[pm.name].overridesKey); + }, + downloadDependency(...args) { + return requirePm((pm) => + packageManagers[pm.name].downloadDependency(...args), + ); + }, + listDependencies(...args) { + return requirePm((pm) => + packageManagers[pm.name].listDependencies(...args), + ); + }, }; } + +const packageManagers: Record = { + npm, + pnpm, + bun, + yarn, +}; diff --git a/src/core/package-managers/npm.ts b/src/core/package-managers/npm.ts new file mode 100644 index 00000000..05c49dd7 --- /dev/null +++ b/src/core/package-managers/npm.ts @@ -0,0 +1,85 @@ +import { Dependency } from '~/types'; +import { WxtPackageManagerImpl } from './types'; +import path from 'node:path'; +import { ensureDir } from 'fs-extra'; + +export const npm: WxtPackageManagerImpl = { + overridesKey: 'overrides', + async downloadDependency(id, downloadDir) { + await ensureDir(downloadDir); + const { execa } = await import('execa'); + const res = await execa('npm', ['pack', id, '--json'], { + cwd: downloadDir, + }); + const packed: PackedDependency[] = JSON.parse(res.stdout); + return path.resolve(downloadDir, packed[0].filename); + }, + async listDependencies(options) { + const args = ['ls', '--json']; + if (options?.all) { + args.push('--depth', 'Infinity'); + } + const { execa } = await import('execa'); + const res = await execa('npm', args, { cwd: options?.cwd }); + const project: NpmListProject = JSON.parse(res.stdout); + + return flattenNpmListOutput([project]); + }, +}; + +export function flattenNpmListOutput(projects: NpmListProject[]): Dependency[] { + const queue: Record[] = projects.flatMap( + (project) => { + const acc: Record[] = []; + if (project.dependencies) acc.push(project.dependencies); + if (project.devDependencies) acc.push(project.devDependencies); + return acc; + }, + ); + const dependencies: Dependency[] = []; + while (queue.length > 0) { + Object.entries(queue.pop()!).forEach(([name, meta]) => { + dependencies.push({ + name, + version: meta.version, + }); + if (meta.dependencies) queue.push(meta.dependencies); + if (meta.devDependencies) queue.push(meta.devDependencies); + }); + } + return dedupeDependencies(dependencies); +} + +export function dedupeDependencies(dependencies: Dependency[]): Dependency[] { + const hashes = new Set(); + return dependencies.filter((dep) => { + const hash = `${dep.name}@${dep.version}`; + if (hashes.has(hash)) { + return false; + } else { + hashes.add(hash); + return true; + } + }); +} + +export interface NpmListProject { + name: string; + dependencies?: Record; + devDependencies?: Record; +} + +export interface NpmListDependency { + version: string; + resolved?: string; + overridden?: boolean; + dependencies?: Record; + devDependencies?: Record; +} + +interface PackedDependency { + id: string; + name: string; + version: string; + filename: string; +} diff --git a/src/core/package-managers/pnpm.ts b/src/core/package-managers/pnpm.ts new file mode 100644 index 00000000..a48decf9 --- /dev/null +++ b/src/core/package-managers/pnpm.ts @@ -0,0 +1,28 @@ +import { NpmListProject, flattenNpmListOutput, npm } from './npm'; +import { WxtPackageManagerImpl } from './types'; + +export const pnpm: WxtPackageManagerImpl = { + overridesKey: 'resolutions', // "pnpm.overrides" has a higher priority, but I don't want to deal with nesting + downloadDependency(...args) { + return npm.downloadDependency(...args); + }, + async listDependencies(options) { + const args = ['ls', '--json']; + if (options?.all) { + args.push('--depth', 'Infinity'); + } + // Helper for testing - since WXT uses pnpm workspaces, folders inside it don't behave like + // standalone projects unless you pass the --ignore-workspace flag. + if ( + typeof process !== 'undefined' && + process.env.WXT_PNPM_IGNORE_WORKSPACE === 'true' + ) { + args.push('--ignore-workspace'); + } + const { execa } = await import('execa'); + const res = await execa('pnpm', args, { cwd: options?.cwd }); + const projects: NpmListProject[] = JSON.parse(res.stdout); + + return flattenNpmListOutput(projects); + }, +}; diff --git a/src/core/package-managers/types.ts b/src/core/package-managers/types.ts new file mode 100644 index 00000000..46ddbf94 --- /dev/null +++ b/src/core/package-managers/types.ts @@ -0,0 +1,6 @@ +import { WxtPackageManager } from '~/types'; + +export type WxtPackageManagerImpl = Pick< + WxtPackageManager, + 'downloadDependency' | 'listDependencies' | 'overridesKey' +>; diff --git a/src/core/package-managers/yarn.ts b/src/core/package-managers/yarn.ts new file mode 100644 index 00000000..ad97e402 --- /dev/null +++ b/src/core/package-managers/yarn.ts @@ -0,0 +1,54 @@ +import { Dependency } from '~/types'; +import { WxtPackageManagerImpl } from './types'; +import { dedupeDependencies, npm } from './npm'; + +export const yarn: WxtPackageManagerImpl = { + overridesKey: 'resolutions', + downloadDependency(...args) { + return npm.downloadDependency(...args); + }, + async listDependencies(options) { + const args = ['list', '--json']; + if (options?.all) { + args.push('--depth', 'Infinity'); + } + const { execa } = await import('execa'); + const res = await execa('yarn', args, { cwd: options?.cwd }); + const tree = res.stdout + .split('\n') + .map((line) => JSON.parse(line)) + .find((line) => line.type === 'tree')?.data as JsonLineTree | undefined; + if (tree == null) throw Error("'yarn list --json' did not output a tree"); + + const queue = [...tree.trees]; + const dependencies: Dependency[] = []; + + while (queue.length > 0) { + const { name: treeName, children } = queue.pop()!; + const match = /(@?\S+)@(\S+)$/.exec(treeName); + if (match) { + const [_, name, version] = match; + dependencies.push({ name, version }); + } + if (children != null) { + queue.push(...children); + } + } + + return dedupeDependencies(dependencies); + }, +}; + +type JsonLine = + | { type: unknown; data: unknown } + | { type: 'tree'; data: JsonLineTree }; + +interface JsonLineTree { + type: 'list'; + trees: Tree[]; +} + +interface Tree { + name: string; + children?: Tree[]; +} diff --git a/src/core/utils/building/resolve-config.ts b/src/core/utils/building/resolve-config.ts index 8e0e78c8..f06fd72c 100644 --- a/src/core/utils/building/resolve-config.ts +++ b/src/core/utils/building/resolve-config.ts @@ -258,6 +258,7 @@ function resolveInternalZipConfig( root: string, mergedConfig: InlineConfig, ): NullablyRequired { + const downloadedPackagesDir = path.resolve(root, '.wxt/local_modules'); return { name: undefined, sourcesTemplate: '{{name}}-{{version}}-sources.zip', @@ -277,6 +278,8 @@ function resolveInternalZipConfig( // From user ...(mergedConfig.zip?.excludeSources ?? []), ], + downloadPackages: mergedConfig.zip?.downloadPackages ?? [], + downloadedPackagesDir, }; } diff --git a/src/core/utils/testing/fake-objects.ts b/src/core/utils/testing/fake-objects.ts index 6c5a947c..033532c7 100644 --- a/src/core/utils/testing/fake-objects.ts +++ b/src/core/utils/testing/fake-objects.ts @@ -286,6 +286,8 @@ export const fakeResolvedConfig = fakeObjectCreator(() => { sourcesRoot: fakeDir(), sourcesTemplate: '{{name}}-sources.zip', name: faker.person.firstName().toLowerCase(), + downloadedPackagesDir: fakeDir(), + downloadPackages: [], }, transformManifest: () => {}, userConfigMetadata: {}, diff --git a/src/core/zip.ts b/src/core/zip.ts index 20f10105..ea511cfe 100644 --- a/src/core/zip.ts +++ b/src/core/zip.ts @@ -10,6 +10,7 @@ import { internalBuild } from '~/core/utils/building'; import { registerWxt, wxt } from './wxt'; import JSZip from 'jszip'; import glob from 'fast-glob'; +import { normalizePath } from './utils/paths'; /** * Build and zip the extension for distribution. @@ -51,6 +52,8 @@ export async function zip(config?: InlineConfig): Promise { // ZIP sources for Firefox if (wxt.config.browser === 'firefox') { + const { overrides, files: downloadedPackages } = + await downloadPrivatePackages(); const sourcesZipFilename = applyTemplate(wxt.config.zip.sourcesTemplate); const sourcesZipPath = path.resolve( wxt.config.outBaseDir, @@ -59,6 +62,12 @@ export async function zip(config?: InlineConfig): Promise { await zipDir(wxt.config.zip.sourcesRoot, sourcesZipPath, { include: wxt.config.zip.includeSources, exclude: wxt.config.zip.excludeSources, + transform(file, content) { + if (file.endsWith('package.json')) { + return addOverridesToPackageJson(content, overrides); + } + }, + additionalFiles: downloadedPackages, }); zipFiles.push(sourcesZipPath); } @@ -83,6 +92,7 @@ async function zipDir( content: string, ) => Promise | string | undefined | void; additionalWork?: (archive: JSZip) => Promise | void; + additionalFiles?: string[]; }, ): Promise { const archive = new JSZip(); @@ -103,7 +113,13 @@ async function zipDir( ) ); }); - for (const file of files) { + const filesToZip = [ + ...files, + ...(options?.additionalFiles ?? []).map((file) => + path.relative(directory, file), + ), + ]; + for (const file of filesToZip) { const absolutePath = path.resolve(directory, file); if (file.endsWith('.json')) { const content = await fs.readFile(absolutePath, 'utf-8'); @@ -120,3 +136,50 @@ async function zipDir( const buffer = await archive.generateAsync({ type: 'base64' }); await fs.writeFile(outputPath, buffer, 'base64'); } + +async function downloadPrivatePackages() { + const overrides: Record = {}; + const files: string[] = []; + + if (wxt.config.zip.downloadPackages.length > 0) { + const _downloadPackages = new Set(wxt.config.zip.downloadPackages); + const allPackages = await wxt.pm.listDependencies({ + all: true, + cwd: wxt.config.root, + }); + const downloadPackages = allPackages.filter((pkg) => + _downloadPackages.has(pkg.name), + ); + + for (const pkg of downloadPackages) { + wxt.logger.info(`Downloading package: ${pkg.name}@${pkg.version}`); + const id = `${pkg.name}@${pkg.version}`; + const tgzPath = await wxt.pm.downloadDependency( + id, + wxt.config.zip.downloadedPackagesDir, + ); + files.push(tgzPath); + overrides[id] = + 'file://./' + normalizePath(path.relative(wxt.config.root, tgzPath)); + } + } + + return { overrides, files }; +} + +function addOverridesToPackageJson( + content: string, + overrides: Record, +): string { + if (Object.keys(overrides).length === 0) return content; + + const oldPackage = JSON.parse(content); + const newPackage = { + ...oldPackage, + [wxt.pm.overridesKey]: { + ...oldPackage[wxt.pm.overridesKey], + ...overrides, + }, + }; + return JSON.stringify(newPackage, null, 2); +} diff --git a/src/types/index.ts b/src/types/index.ts index 7bd130d8..930bc7d6 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -172,6 +172,31 @@ export interface InlineConfig { * ] */ excludeSources?: string[]; + /** + * The Firefox review process requires the extension be buildable from source to make reviewing + * easier. This field allows you to use private packages without exposing your auth tokens. + * + * Just list the name of all the packages you want to download and include in the sources zip. + * Usually, these will be private packages behind auth tokens, but they don't have to be. + * + * All packages listed here will be downloaded to in `.wxt/local_modules/` and an `overrides` or + * `resolutions` field (depending on your package manager) will be added to the `package.json`, + * pointing to the downloaded packages. + * + * > ***DO NOT include versions or version filters.*** Just the package name. If multiple + * > versions of a package are present in the project, all versions will be downloaded and + * > referenced in the package.json correctly. + * + * @default [] + * + * @example + * // Correct: + * ["@scope/package-name", "package-name"] + * + * // Incorrect, don't include versions!!! + * ["@scope/package-name@1.1.3", "package-name@^2"] + */ + downloadPackages?: string[]; }; /** @@ -999,6 +1024,8 @@ export interface ResolvedConfig { includeSources: string[]; excludeSources: string[]; sourcesRoot: string; + downloadedPackagesDir: string; + downloadPackages: string[]; }; transformManifest: (manifest: Manifest.WebExtensionManifest) => void; analysis: { @@ -1100,4 +1127,31 @@ export interface WxtPackageManager extends Nypm.PackageManager { ensureDependencyInstalled: typeof Nypm.ensureDependencyInstalled; installDependencies: typeof Nypm.installDependencies; removeDependency: typeof Nypm.removeDependency; + /** + * Download a package's TGZ file and move it into the `downloadDir`. Use's `npm pack `, so + * you must have setup authorization in `.npmrc` file, regardless of the package manager used. + * + * @param id Name of the package to download, can include a version (like `wxt@0.17.1`) + * @param downloadDir Where to store the package. + * @returns Absolute path to downloaded file. + */ + downloadDependency: (id: string, downloadDir: string) => Promise; + /** + * Run `npm ls`, `pnpm ls`, or `bun pm ls`, or `yarn list` and return the results. + * + * WARNING: Yarn always returns all dependencies + */ + listDependencies: (options?: { + cwd?: string; + all?: boolean; + }) => Promise; + /** + * Key used to override package versions. Sometimes called "resolutions". + */ + overridesKey: string; +} + +export interface Dependency { + name: string; + version: string; }