feat: Download and bundle remote URL imports

README.md

@@ -2,14 +2,18 @@
 
 > Next generation framework for building web extensions. Powered by [Vite](https://vitejs.dev/)
 
+Name ideas:
+
+cyfer, wavex, webext, aura, iron, reyna, aggron, wxt, yiga
+
 ## Features
 
 - [x] Supports all browsers
-- [ ] MV2 & MV3 support
-- [ ] Directory based entrypoints
+- [x] MV2 & MV3 support
+- [x] Directory based entrypoints
 - [ ] Great DX: auto-imports, TypeScript, HMR, and fast reload for content scripts
-- [ ] Infer permissions from code
+- [ ] ~~_Infer permissions from code_~~
 - [ ] Automated publishing
-- [ ] Download and bundle remote dependencies
+- [x] Download and bundle remote dependencies
 - [ ] Supports all major frontend frameworks (Vue, React, Svelte)
 - [ ] Bundle analysis

demo/entrypoints/options/main.ts

@@ -1,3 +1,4 @@
 import browser from 'webextension-polyfill';
+import 'url:https://stats.aklinker1.io/umami.js';
 
 console.log(browser.runtime.id);

src/types/internal.ts

@@ -24,6 +24,12 @@ export interface InternalConfig {
   logger: Logger;
   vite: vite.InlineConfig;
   manifest: UserManifest;
+  fsCache: FsCache;
 }
 
 export type EntrypointGroup = Entrypoint | Entrypoint[];
+
+export interface FsCache {
+  set(key: string, value: string): Promise<void>;
+  get(key: string): Promise<string | undefined>;
+}

src/utils/createFsCache.ts

@@ -0,0 +1,30 @@
+import fs, { ensureDir } from 'fs-extra';
+import { FsCache } from '../types';
+import { dirname, resolve } from 'path';
+
+/**
+ * A basic file system cache stored at `<srcDir>/.exvite/cache/<key>`. Just caches a string in a
+ * file for the given key.
+ *
+ * @param srcDir Absolute path to source directory. See `InternalConfig.srcDir`
+ */
+export function createFsCache(srcDir: string): FsCache {
+  const getPath = (key: string) =>
+    resolve(srcDir, '.exvite/cache', encodeURIComponent(key));
+
+  return {
+    async set(key: string, value: string): Promise<void> {
+      const path = getPath(key);
+      await ensureDir(dirname(path));
+      await fs.writeFile(path, value, 'utf-8');
+    },
+    async get(key: string): Promise<string | undefined> {
+      const path = getPath(key);
+      try {
+        return await fs.readFile(path, 'utf-8');
+      } catch {
+        return undefined;
+      }
+    },
+  };
+}

src/utils/getInternalConfig.ts

@@ -4,6 +4,7 @@ import * as vite from 'vite';
 import { consola } from 'consola';
 import { importTsFile } from './importTsFile';
 import * as plugins from '../vite-plugins';
+import { createFsCache } from './createFsCache';
 
 /**
  * Given an inline config, discover the config file if necessary, merge the results, resolve any
@@ -31,6 +32,7 @@ export async function getInternalConfig(
     '.output',
     `${browser}-mv${manifestVersion}`,
   );
+  const logger = config.logger ?? consola;
 
   const baseConfig: InternalConfig = {
     root,
@@ -42,9 +44,10 @@ export async function getInternalConfig(
     mode,
     command,
     outDir,
-    logger: config.logger ?? consola,
+    logger,
     vite: config.vite ?? {},
     manifest: config.manifest ?? {},
+    fsCache: createFsCache(srcDir),
   };
 
   // Load user config from file

src/utils/network.ts

@@ -0,0 +1,52 @@
+import dns from 'node:dns';
+import { withTimeout } from './promises';
+import { InternalConfig } from '../types';
+
+function isOffline(): Promise<boolean> {
+  const isOffline = new Promise<boolean>((res) => {
+    dns.resolve('google.com', (err) => {
+      if (err == null) {
+        res(false);
+      } else {
+        res(true);
+      }
+    });
+  });
+  return withTimeout(isOffline, 1e3).catch(() => true);
+}
+
+export async function isOnline(): Promise<boolean> {
+  const offline = await isOffline();
+  return !offline;
+}
+
+/**
+ * Fetches a URL with a simple GET request. Grabs it from cache if it doesn't exist, or throws an
+ * error if it can't be resolved via the network or cache.
+ */
+export async function fetchCached(
+  url: string,
+  config: InternalConfig,
+): Promise<string> {
+  let content: string = '';
+
+  if (await isOnline()) {
+    const res = await fetch(url);
+    if (res.status < 300) {
+      content = await res.text();
+      await config.fsCache.set(url, content);
+    } else {
+      config.logger.debug(
+        `Failed to download "${url}", falling back to cache...`,
+      );
+    }
+  }
+
+  if (!content) content = (await config.fsCache.get(url)) ?? '';
+  if (!content)
+    throw Error(
+      `Offline and "${url}" has not been cached. Try again when online.`,
+    );
+
+  return content;
+}

src/utils/promises.ts

@@ -0,0 +1,24 @@
+/**
+ * Add a timeout to a promise.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  duration: number,
+): Promise<T> {
+  return new Promise((res, rej) => {
+    const timeout = setTimeout(() => {
+      rej(`Promise timed out after ${duration}ms`);
+    }, duration);
+    promise
+      .then(res)
+      .catch(rej)
+      .finally(() => clearTimeout(timeout));
+  });
+}
+
+/**
+ * @deprecated Don't use in production, just for testing and slowing things down.
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((res) => setTimeout(res, ms));
+}

src/vite-plugins/download.ts

@@ -1,16 +1,26 @@
 import { Plugin } from 'vite';
 import { InternalConfig } from '../types';
+import { fetchCached } from '../utils/network';
 
 /**
- * Downloads any URL imports into virtual modules, like Google Analytics, so they are bundled with
- * the extension instead of depending on remote code.
+ * Downloads any URL imports, like Google Analytics, into virtual modules so they are bundled with
+ * the extension instead of depending on remote code at runtime.
  *
  * @example
- * import "https://google-tagmanager.com/gtag?id=XYZ";
+ * import "url:https://google-tagmanager.com/gtag?id=XYZ";
  */
 export function download(config: InternalConfig): Plugin {
-  config.logger.warn('Not implemented: download plugin');
   return {
     name: 'exvite:download',
+    resolveId(id) {
+      if (id.startsWith('url:')) return '\0' + id;
+    },
+    async load(id) {
+      if (!id.startsWith('\0url:')) return;
+
+      // Load file from network or cache
+      const url = id.replace('\0url:', '');
+      return await fetchCached(url, config);
+    },
   };
 }