diff --git a/e2e/tests/user-config.test.ts b/e2e/tests/user-config.test.ts
index c8275888..0672988d 100644
--- a/e2e/tests/user-config.test.ts
+++ b/e2e/tests/user-config.test.ts
@@ -83,7 +83,7 @@ describe('User Config', () => {
     expect(output).toMatchInlineSnapshot(`
       ".output/chrome-mv3/manifest.json
       ----------------------------------------
-      {\\"manifest_version\\":3,\\"name\\":\\"E2E Extension\\",\\"description\\":\\"Example description\\",\\"version\\":\\"0.0.0\\",\\"example_customization\\":[\\"production\\",\\"chrome\\",\\"3\\",\\"build\\"]}"
+      {\\"manifest_version\\":3,\\"name\\":\\"E2E Extension\\",\\"description\\":\\"Example description\\",\\"version\\":\\"0.0.0\\",\\"example_customization\\":[\\"3\\",\\"build\\",\\"production\\",\\"chrome\\"]}"
     `);
   });
 
diff --git a/package.json b/package.json
index a8a6471b..e336deb1 100644
--- a/package.json
+++ b/package.json
@@ -87,6 +87,7 @@
     "async-mutex": "^0.4.0",
     "c12": "^1.5.1",
     "cac": "^6.7.14",
+    "chokidar": "^3.5.3",
     "consola": "^3.2.3",
     "defu": "^6.1.3",
     "esbuild": "^0.19.5",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4ab96311..48a559ee 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -29,6 +29,9 @@ importers:
       cac:
         specifier: ^6.7.14
         version: 6.7.14
+      chokidar:
+        specifier: ^3.5.3
+        version: 3.5.3
       consola:
         specifier: ^3.2.3
         version: 3.2.3
diff --git a/src/core/builders/vite/index.ts b/src/core/builders/vite/index.ts
new file mode 100644
index 00000000..21cb223b
--- /dev/null
+++ b/src/core/builders/vite/index.ts
@@ -0,0 +1,218 @@
+import type * as vite from 'vite';
+import {
+  BuildStepOutput,
+  Entrypoint,
+  InlineConfig,
+  InternalConfig,
+  UserConfig,
+  WxtBuilder,
+  WxtBuilderServer,
+} from '~/types';
+import * as wxtPlugins from './plugins';
+import { getEntrypointBundlePath } from '~/core/utils/entrypoints';
+
+export async function craeteViteBuilder(
+  inlineConfig: InlineConfig,
+  userConfig: UserConfig,
+  wxtConfig: Omit<InternalConfig, 'builder'>,
+): Promise<WxtBuilder> {
+  const vite = await import('vite');
+
+  /**
+   * Returns the base vite config shared by all builds based on the inline and user config.
+   */
+  const getBaseConfig = async () => {
+    const resolvedInlineConfig =
+      (await inlineConfig.vite?.(wxtConfig.env)) ?? {};
+    const resolvedUserConfig = (await userConfig.vite?.(wxtConfig.env)) ?? {};
+
+    const config: vite.InlineConfig = vite.mergeConfig(
+      resolvedUserConfig,
+      resolvedInlineConfig,
+    );
+
+    config.root = wxtConfig.root;
+    config.configFile = false;
+    config.logLevel = 'warn';
+    config.mode = wxtConfig.mode;
+
+    config.build ??= {};
+    config.build.outDir = wxtConfig.outDir;
+    config.build.emptyOutDir = false;
+
+    config.plugins ??= [];
+    config.plugins.push(
+      wxtPlugins.download(wxtConfig),
+      wxtPlugins.devHtmlPrerender(wxtConfig),
+      wxtPlugins.unimport(wxtConfig),
+      wxtPlugins.virtualEntrypoint('background', wxtConfig),
+      wxtPlugins.virtualEntrypoint('content-script', wxtConfig),
+      wxtPlugins.virtualEntrypoint('unlisted-script', wxtConfig),
+      wxtPlugins.devServerGlobals(wxtConfig),
+      wxtPlugins.tsconfigPaths(wxtConfig),
+      wxtPlugins.noopBackground(),
+      wxtPlugins.globals(wxtConfig),
+      wxtPlugins.excludeBrowserPolyfill(wxtConfig),
+    );
+    if (wxtConfig.analysis.enabled) {
+      config.plugins.push(wxtPlugins.bundleAnalysis());
+    }
+
+    return config;
+  };
+
+  /**
+   * Return the basic config for building an entrypoint in [lib mode](https://vitejs.dev/guide/build.html#library-mode).
+   */
+  const getLibModeConfig = (entrypoint: Entrypoint): vite.InlineConfig => {
+    const isVirtual = [
+      'background',
+      'content-script',
+      'unlisted-script',
+    ].includes(entrypoint.type);
+    const entry = isVirtual
+      ? `virtual:wxt-${entrypoint.type}?${entrypoint.inputPath}`
+      : entrypoint.inputPath;
+
+    const plugins: NonNullable<vite.UserConfig['plugins']> = [
+      wxtPlugins.entrypointGroupGlobals(entrypoint),
+    ];
+    if (
+      entrypoint.type === 'content-script-style' ||
+      entrypoint.type === 'unlisted-style'
+    ) {
+      plugins.push(wxtPlugins.cssEntrypoints(entrypoint, wxtConfig));
+    }
+
+    const libMode: vite.UserConfig = {
+      mode: wxtConfig.mode,
+      plugins,
+      build: {
+        lib: {
+          entry,
+          formats: ['iife'],
+          name: '_',
+          fileName: entrypoint.name,
+        },
+        rollupOptions: {
+          output: {
+            // There's only a single output for this build, so we use the desired bundle path for the
+            // entry output (like "content-scripts/overlay.js")
+            entryFileNames: getEntrypointBundlePath(
+              entrypoint,
+              wxtConfig.outDir,
+              '.js',
+            ),
+            // Output content script CSS to `content-scripts/`, but all other scripts are written to
+            // `assets/`.
+            assetFileNames: ({ name }) => {
+              if (
+                entrypoint.type === 'content-script' &&
+                name?.endsWith('css')
+              ) {
+                return `content-scripts/${entrypoint.name}.[ext]`;
+              } else {
+                return `assets/${entrypoint.name}.[ext]`;
+              }
+            },
+          },
+        },
+      },
+      define: {
+        // See https://github.com/aklinker1/vite-plugin-web-extension/issues/96
+        'process.env.NODE_ENV': JSON.stringify(wxtConfig.mode),
+      },
+    };
+    return libMode;
+  };
+
+  /**
+   * Return the basic config for building multiple entrypoints in [multi-page mode](https://vitejs.dev/guide/build.html#multi-page-app).
+   */
+  const getMultiPageConfig = (entrypoints: Entrypoint[]): vite.InlineConfig => {
+    return {
+      mode: wxtConfig.mode,
+      plugins: [
+        wxtPlugins.multipageMove(entrypoints, wxtConfig),
+        wxtPlugins.entrypointGroupGlobals(entrypoints),
+      ],
+      build: {
+        rollupOptions: {
+          input: entrypoints.reduce<Record<string, string>>((input, entry) => {
+            input[entry.name] = entry.inputPath;
+            return input;
+          }, {}),
+          output: {
+            // Include a hash to prevent conflicts
+            chunkFileNames: 'chunks/[name]-[hash].js',
+            // Include a hash to prevent conflicts
+            entryFileNames: 'chunks/[name]-[hash].js',
+            // We can't control the "name", so we need a hash to prevent conflicts
+            assetFileNames: 'assets/[name]-[hash].[ext]',
+          },
+        },
+      },
+      define: {},
+    };
+  };
+
+  return {
+    name: 'Vite',
+    version: vite.version,
+    async build(group) {
+      const buildConfig = vite.mergeConfig(
+        await getBaseConfig(),
+        Array.isArray(group)
+          ? getMultiPageConfig(group)
+          : getLibModeConfig(group),
+      );
+      const result = await vite.build(buildConfig);
+      return {
+        entrypoints: group,
+        chunks: getBuildOutputChunks(result),
+      };
+    },
+    async createServer(info) {
+      const serverConfig: vite.InlineConfig = {
+        server: {
+          port: info.port,
+          strictPort: true,
+          host: info.hostname,
+          origin: info.origin,
+        },
+      };
+      const baseConfig = await getBaseConfig();
+      const viteServer = await vite.createServer(
+        vite.mergeConfig(baseConfig, serverConfig),
+      );
+
+      const server: WxtBuilderServer = {
+        async listen() {
+          await viteServer.listen(info.port);
+        },
+        transformHtml(...args) {
+          return viteServer.transformIndexHtml(...args);
+        },
+        ws: {
+          send(message, payload) {
+            return viteServer.ws.send(message, payload);
+          },
+          on(message, cb) {
+            viteServer.ws.on(message, cb);
+          },
+        },
+        watcher: viteServer.watcher,
+      };
+
+      return server;
+    },
+  };
+}
+
+function getBuildOutputChunks(
+  result: Awaited<ReturnType<typeof vite.build>>,
+): BuildStepOutput['chunks'] {
+  if ('on' in result) throw Error('wxt does not support vite watch mode.');
+  if (Array.isArray(result)) return result.flatMap(({ output }) => output);
+  return result.output;
+}
diff --git a/src/core/vite-plugins/bundleAnalysis.ts b/src/core/builders/vite/plugins/bundleAnalysis.ts
similarity index 100%
rename from src/core/vite-plugins/bundleAnalysis.ts
rename to src/core/builders/vite/plugins/bundleAnalysis.ts
diff --git a/src/core/vite-plugins/cssEntrypoints.ts b/src/core/builders/vite/plugins/cssEntrypoints.ts
similarity index 96%
rename from src/core/vite-plugins/cssEntrypoints.ts
rename to src/core/builders/vite/plugins/cssEntrypoints.ts
index dc813b66..4796a027 100644
--- a/src/core/vite-plugins/cssEntrypoints.ts
+++ b/src/core/builders/vite/plugins/cssEntrypoints.ts
@@ -14,7 +14,7 @@ import { getEntrypointBundlePath } from '~/core/utils/entrypoints';
  */
 export function cssEntrypoints(
   entrypoint: Entrypoint,
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): vite.Plugin {
   return {
     name: 'wxt:css-entrypoint',
diff --git a/src/core/vite-plugins/devHtmlPrerender.ts b/src/core/builders/vite/plugins/devHtmlPrerender.ts
similarity index 94%
rename from src/core/vite-plugins/devHtmlPrerender.ts
rename to src/core/builders/vite/plugins/devHtmlPrerender.ts
index 147f232f..83d52d32 100644
--- a/src/core/vite-plugins/devHtmlPrerender.ts
+++ b/src/core/builders/vite/plugins/devHtmlPrerender.ts
@@ -2,7 +2,7 @@ import * as vite from 'vite';
 import { InternalConfig } from '~/types';
 import { getEntrypointName } from '~/core/utils/entrypoints';
 import { parseHTML } from 'linkedom';
-import { dirname, isAbsolute, relative, resolve } from 'path';
+import { dirname, isAbsolute, relative, resolve } from 'node:path';
 
 // Cache the preamble script for all devHtmlPrerender plugins, not just one
 let reactRefreshPreamble = '';
@@ -10,7 +10,9 @@ let reactRefreshPreamble = '';
 /**
  * Pre-renders the HTML entrypoints when building the extension to connect to the dev server.
  */
-export function devHtmlPrerender(config: InternalConfig): vite.PluginOption {
+export function devHtmlPrerender(
+  config: Omit<InternalConfig, 'builder'>,
+): vite.PluginOption {
   const htmlReloadId = '@wxt/reload-html';
   const resolvedHtmlReloadId = resolve(
     config.root,
@@ -86,11 +88,7 @@ export function devHtmlPrerender(config: InternalConfig): vite.PluginOption {
         const originalUrl = `${server.origin}${ctx.path}`;
         const name = getEntrypointName(config.entrypointsDir, ctx.filename);
         const url = `${server.origin}/${name}.html`;
-        const serverHtml = await server.transformIndexHtml(
-          url,
-          html,
-          originalUrl,
-        );
+        const serverHtml = await server.transformHtml(url, html, originalUrl);
         const { document } = parseHTML(serverHtml);
 
         // React pages include a preamble as an unsafe-inline type="module" script to enable fast refresh, as shown here:
diff --git a/src/core/vite-plugins/devServerGlobals.ts b/src/core/builders/vite/plugins/devServerGlobals.ts
similarity index 51%
rename from src/core/vite-plugins/devServerGlobals.ts
rename to src/core/builders/vite/plugins/devServerGlobals.ts
index 238c63e7..6572fc34 100644
--- a/src/core/vite-plugins/devServerGlobals.ts
+++ b/src/core/builders/vite/plugins/devServerGlobals.ts
@@ -4,20 +4,19 @@ import { InternalConfig } from '~/types';
 /**
  * Defines global constants about the dev server. Helps scripts connect to the server's web socket.
  */
-export function devServerGlobals(internalConfig: InternalConfig): Plugin {
+export function devServerGlobals(
+  config: Omit<InternalConfig, 'builder'>,
+): Plugin {
   return {
     name: 'wxt:dev-server-globals',
     config() {
-      if (internalConfig.server == null || internalConfig.command == 'build')
-        return;
+      if (config.server == null || config.command == 'build') return;
 
       return {
         define: {
           __DEV_SERVER_PROTOCOL__: JSON.stringify('ws:'),
-          __DEV_SERVER_HOSTNAME__: JSON.stringify(
-            internalConfig.server.hostname,
-          ),
-          __DEV_SERVER_PORT__: JSON.stringify(internalConfig.server.port),
+          __DEV_SERVER_HOSTNAME__: JSON.stringify(config.server.hostname),
+          __DEV_SERVER_PORT__: JSON.stringify(config.server.port),
         },
       };
     },
diff --git a/src/core/vite-plugins/download.ts b/src/core/builders/vite/plugins/download.ts
similarity index 90%
rename from src/core/vite-plugins/download.ts
rename to src/core/builders/vite/plugins/download.ts
index 6d5c5fe8..8c90e96e 100644
--- a/src/core/vite-plugins/download.ts
+++ b/src/core/builders/vite/plugins/download.ts
@@ -9,7 +9,7 @@ import { fetchCached } from '~/core/utils/network';
  * @example
  * import "url:https://google-tagmanager.com/gtag?id=XYZ";
  */
-export function download(config: InternalConfig): Plugin {
+export function download(config: Omit<InternalConfig, 'builder'>): Plugin {
   return {
     name: 'wxt:download',
     resolveId(id) {
diff --git a/src/core/builders/vite/plugins/entrypointGroupGlobals.ts b/src/core/builders/vite/plugins/entrypointGroupGlobals.ts
new file mode 100644
index 00000000..d6e3de83
--- /dev/null
+++ b/src/core/builders/vite/plugins/entrypointGroupGlobals.ts
@@ -0,0 +1,24 @@
+import * as vite from 'vite';
+import { EntrypointGroup } from '~/types';
+import { getEntrypointGlobals } from '~/core/utils/globals';
+
+/**
+ * Define a set of global variables specific to an entrypoint.
+ */
+export function entrypointGroupGlobals(
+  entrypointGroup: EntrypointGroup,
+): vite.PluginOption {
+  return {
+    name: 'wxt:entrypoint-group-globals',
+    config() {
+      const define: vite.InlineConfig['define'] = {};
+      let name = Array.isArray(entrypointGroup) ? 'html' : entrypointGroup.name;
+      for (const global of getEntrypointGlobals(name)) {
+        define[global.name] = JSON.stringify(global.value);
+      }
+      return {
+        define,
+      };
+    },
+  };
+}
diff --git a/src/core/vite-plugins/excludeBrowserPolyfill.ts b/src/core/builders/vite/plugins/excludeBrowserPolyfill.ts
similarity index 89%
rename from src/core/vite-plugins/excludeBrowserPolyfill.ts
rename to src/core/builders/vite/plugins/excludeBrowserPolyfill.ts
index d6bf9a94..c0805106 100644
--- a/src/core/vite-plugins/excludeBrowserPolyfill.ts
+++ b/src/core/builders/vite/plugins/excludeBrowserPolyfill.ts
@@ -6,7 +6,9 @@ import * as vite from 'vite';
  * `webextension-polyfill` module to a virtual module and exporting the `chrome` global from the
  * virtual module.
  */
-export function excludeBrowserPolyfill(config: InternalConfig): vite.Plugin {
+export function excludeBrowserPolyfill(
+  config: Omit<InternalConfig, 'builder'>,
+): vite.Plugin {
   const virtualId = 'virtual:wxt-webextension-polyfill-disabled';
 
   return {
diff --git a/src/core/vite-plugins/globals.ts b/src/core/builders/vite/plugins/globals.ts
similarity index 71%
rename from src/core/vite-plugins/globals.ts
rename to src/core/builders/vite/plugins/globals.ts
index 019ae0dc..5725c25e 100644
--- a/src/core/vite-plugins/globals.ts
+++ b/src/core/builders/vite/plugins/globals.ts
@@ -1,8 +1,10 @@
 import * as vite from 'vite';
 import { InternalConfig } from '~/types';
-import { getGlobals } from '../utils/globals';
+import { getGlobals } from '~/core/utils/globals';
 
-export function globals(config: InternalConfig): vite.PluginOption {
+export function globals(
+  config: Omit<InternalConfig, 'builder'>,
+): vite.PluginOption {
   return {
     name: 'wxt:globals',
     config() {
diff --git a/src/core/vite-plugins/index.ts b/src/core/builders/vite/plugins/index.ts
similarity index 92%
rename from src/core/vite-plugins/index.ts
rename to src/core/builders/vite/plugins/index.ts
index da49d5f5..4bba7677 100644
--- a/src/core/vite-plugins/index.ts
+++ b/src/core/builders/vite/plugins/index.ts
@@ -12,3 +12,4 @@ export * from './globals';
 export * from './webextensionPolyfillAlias';
 export * from './webextensionPolyfillInlineDeps';
 export * from './excludeBrowserPolyfill';
+export * from './entrypointGroupGlobals';
diff --git a/src/core/vite-plugins/multipageMove.ts b/src/core/builders/vite/plugins/multipageMove.ts
similarity index 77%
rename from src/core/vite-plugins/multipageMove.ts
rename to src/core/builders/vite/plugins/multipageMove.ts
index 562ed8d3..4800eb11 100644
--- a/src/core/vite-plugins/multipageMove.ts
+++ b/src/core/builders/vite/plugins/multipageMove.ts
@@ -1,6 +1,6 @@
 import * as vite from 'vite';
 import { Entrypoint, InternalConfig } from '~/types';
-import { dirname, extname, resolve } from 'node:path';
+import { dirname, extname, resolve, join } from 'node:path';
 import { getEntrypointBundlePath } from '~/core/utils/entrypoints';
 import fs, { ensureDir } from 'fs-extra';
 import { normalizePath } from '~/core/utils/paths';
@@ -20,7 +20,7 @@ import { normalizePath } from '~/core/utils/paths';
  */
 export function multipageMove(
   entrypoints: Entrypoint[],
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): vite.Plugin {
   return {
     name: 'wxt:multipage-move',
@@ -67,6 +67,30 @@ export function multipageMove(
         delete bundle[oldBundlePath];
         bundle[newBundlePath] = renamedChunk;
       }
+
+      // Remove directories that were created
+      // TODO: Optimize and only delete old path directories
+      removeEmptyDirs(config.outDir);
     },
   };
 }
+
+/**
+ * Recursively remove all directories that are empty/
+ */
+export async function removeEmptyDirs(dir: string): Promise<void> {
+  const files = await fs.readdir(dir);
+  for (const file of files) {
+    const filePath = join(dir, file);
+    const stats = await fs.stat(filePath);
+    if (stats.isDirectory()) {
+      await removeEmptyDirs(filePath);
+    }
+  }
+
+  try {
+    await fs.rmdir(dir);
+  } catch {
+    // noop on failure - this means the directory was not empty.
+  }
+}
diff --git a/src/core/vite-plugins/noopBackground.ts b/src/core/builders/vite/plugins/noopBackground.ts
similarity index 89%
rename from src/core/vite-plugins/noopBackground.ts
rename to src/core/builders/vite/plugins/noopBackground.ts
index 84fcd79f..d1da0785 100644
--- a/src/core/vite-plugins/noopBackground.ts
+++ b/src/core/builders/vite/plugins/noopBackground.ts
@@ -1,4 +1,5 @@
 import { Plugin } from 'vite';
+import { VIRTUAL_NOOP_BACKGROUND_MODULE_ID } from '~/core/utils/constants';
 
 /**
  * In dev mode, if there's not a background script listed, we need to add one.
@@ -20,5 +21,3 @@ export function noopBackground(): Plugin {
     },
   };
 }
-
-export const VIRTUAL_NOOP_BACKGROUND_MODULE_ID = 'virtual:user-background';
diff --git a/src/core/vite-plugins/tsconfigPaths.ts b/src/core/builders/vite/plugins/tsconfigPaths.ts
similarity index 71%
rename from src/core/vite-plugins/tsconfigPaths.ts
rename to src/core/builders/vite/plugins/tsconfigPaths.ts
index 33126e36..64966d3d 100644
--- a/src/core/vite-plugins/tsconfigPaths.ts
+++ b/src/core/builders/vite/plugins/tsconfigPaths.ts
@@ -1,7 +1,9 @@
 import { InternalConfig } from '~/types';
 import * as vite from 'vite';
 
-export function tsconfigPaths(config: InternalConfig): vite.Plugin {
+export function tsconfigPaths(
+  config: Omit<InternalConfig, 'builder'>,
+): vite.Plugin {
   return {
     name: 'wxt:aliases',
     async config() {
diff --git a/src/core/vite-plugins/unimport.ts b/src/core/builders/vite/plugins/unimport.ts
similarity index 90%
rename from src/core/vite-plugins/unimport.ts
rename to src/core/builders/vite/plugins/unimport.ts
index 498cf160..942bdf8b 100644
--- a/src/core/vite-plugins/unimport.ts
+++ b/src/core/builders/vite/plugins/unimport.ts
@@ -16,7 +16,9 @@ const ENABLED_EXTENSIONS = new Set([
 /**
  * Inject any global imports defined by unimport
  */
-export function unimport(config: InternalConfig): vite.PluginOption {
+export function unimport(
+  config: Omit<InternalConfig, 'builder'>,
+): vite.PluginOption {
   const options = getUnimportOptions(config);
   if (options === false) return [];
 
diff --git a/src/core/vite-plugins/virtualEntrypoint.ts b/src/core/builders/vite/plugins/virtualEntrypoint.ts
similarity index 96%
rename from src/core/vite-plugins/virtualEntrypoint.ts
rename to src/core/builders/vite/plugins/virtualEntrypoint.ts
index 1214f1c7..d6e00132 100644
--- a/src/core/vite-plugins/virtualEntrypoint.ts
+++ b/src/core/builders/vite/plugins/virtualEntrypoint.ts
@@ -9,7 +9,7 @@ import { normalizePath } from '~/core/utils/paths';
  */
 export function virtualEntrypoint(
   type: Entrypoint['type'],
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): Plugin {
   const virtualId = `virtual:wxt-${type}?`;
   const resolvedVirtualId = `\0${virtualId}`;
diff --git a/src/core/vite-plugins/webextensionPolyfillAlias.ts b/src/core/builders/vite/plugins/webextensionPolyfillAlias.ts
similarity index 93%
rename from src/core/vite-plugins/webextensionPolyfillAlias.ts
rename to src/core/builders/vite/plugins/webextensionPolyfillAlias.ts
index 303fb933..d98fec3f 100644
--- a/src/core/vite-plugins/webextensionPolyfillAlias.ts
+++ b/src/core/builders/vite/plugins/webextensionPolyfillAlias.ts
@@ -8,7 +8,7 @@ import { InternalConfig } from '~/types';
  * This should only be used during tests.
  */
 export function webextensionPolyfillAlias(
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): vite.PluginOption {
   return {
     name: 'wxt:webextension-polyfill-test-alias',
diff --git a/src/core/vite-plugins/webextensionPolyfillInlineDeps.ts b/src/core/builders/vite/plugins/webextensionPolyfillInlineDeps.ts
similarity index 100%
rename from src/core/vite-plugins/webextensionPolyfillInlineDeps.ts
rename to src/core/builders/vite/plugins/webextensionPolyfillInlineDeps.ts
diff --git a/src/core/create-server.ts b/src/core/create-server.ts
index 9ac81454..002dc595 100644
--- a/src/core/create-server.ts
+++ b/src/core/create-server.ts
@@ -3,10 +3,9 @@ import {
   EntrypointGroup,
   InlineConfig,
   InternalConfig,
+  ServerInfo,
   WxtDevServer,
 } from '~/types';
-import * as vite from 'vite';
-import type { Scripting } from 'webextension-polyfill';
 import {
   getEntrypointBundlePath,
   getEntrypointOutputFile,
@@ -38,35 +37,103 @@ import { relative } from 'node:path';
  * await server.start();
  */
 export async function createServer(
-  config?: InlineConfig,
+  inlineConfig?: InlineConfig,
 ): Promise<WxtDevServer> {
-  const serverInfo = await getServerInfo();
-
-  const getLatestInternalConfig = async () => {
-    return getInternalConfig(
-      {
-        ...config,
-        vite: () => serverInfo.viteServerConfig,
-      },
-      'serve',
-    );
+  const port = await getPort();
+  const hostname = 'localhost';
+  const origin = `http://${hostname}:${port}`;
+  const serverInfo: ServerInfo = {
+    port,
+    hostname,
+    origin,
   };
 
-  let internalConfig = await getLatestInternalConfig();
-  const server = await setupServer(serverInfo, internalConfig);
-  internalConfig.server = server;
+  // Server instance must be created first so its reference can be added to the internal config used
+  // to pre-render entrypoints
+  const server: WxtDevServer = {
+    ...serverInfo,
+    watcher: undefined as any, // Filled out later down below
+    ws: undefined as any, // Filled out later down below
+    currentOutput: undefined as any, // Filled out later down below
+    async start() {
+      await builderServer.listen();
+      config.logger.success(`Started dev server @ ${serverInfo.origin}`);
 
+      // Build after starting the dev server so it can be used to transform HTML files
+      server.currentOutput = await internalBuild(config);
+
+      // Open browser after everything is ready to go.
+      await runner.openBrowser(config);
+    },
+    transformHtml(url, html, originalUrl) {
+      return builderServer.transformHtml(url, html, originalUrl);
+    },
+    reloadContentScript(contentScript) {
+      server.ws.send('wxt:reload-content-script', contentScript);
+    },
+    reloadPage(path) {
+      server.ws.send('wxt:reload-page', path);
+    },
+    reloadExtension() {
+      server.ws.send('wxt:reload-extension');
+    },
+  };
+
+  const getLatestConfig = () =>
+    getInternalConfig(inlineConfig ?? {}, 'serve', server);
+  let config = await getLatestConfig();
+
+  const [runner, builderServer] = await Promise.all([
+    createExtensionRunner(config),
+    config.builder.createServer(server),
+  ]);
+
+  server.watcher = builderServer.watcher;
+  server.ws = builderServer.ws;
+
+  // Register content scripts for the first time after the background starts up since they're not
+  // listed in the manifest
+  server.ws.on('wxt:background-initialized', () => {
+    reloadContentScripts(server.currentOutput.steps, config, server);
+  });
+
+  // Listen for file changes and reload different parts of the extension accordingly
+  const reloadOnChange = createFileReloader({
+    server,
+    getLatestConfig,
+    updateConfig(newConfig) {
+      config = newConfig;
+    },
+  });
+  server.watcher.on('all', reloadOnChange);
+
+  return server;
+}
+
+async function getPort(): Promise<number> {
+  const { default: getPort, portNumbers } = await import('get-port');
+  return await getPort({ port: portNumbers(3000, 3010) });
+}
+
+/**
+ * Returns a function responsible for reloading different parts of the extension when a file
+ * changes.
+ */
+function createFileReloader(options: {
+  server: WxtDevServer;
+  getLatestConfig: () => Promise<InternalConfig>;
+  updateConfig: (config: InternalConfig) => void;
+}) {
+  const { server, getLatestConfig, updateConfig } = options;
   const fileChangedMutex = new Mutex();
   const changeQueue: Array<[string, string]> = [];
 
-  server.ws.on('wxt:background-initialized', () => {
-    // Register content scripts for the first time since they're not listed in the manifest
-    reloadContentScripts(server.currentOutput.steps, internalConfig, server);
-  });
+  return async (event: string, path: string) => {
+    const config = await getLatestConfig();
+    updateConfig(config);
 
-  server.watcher.on('all', async (event, path, _stats) => {
     // Here, "path" is a non-normalized path (ie: C:\\users\\... instead of C:/users/...)
-    if (path.startsWith(internalConfig.outBaseDir)) return;
+    if (path.startsWith(config.outBaseDir)) return;
     changeQueue.push([event, path]);
 
     await fileChangedMutex.runExclusive(async () => {
@@ -77,25 +144,23 @@ export async function createServer(
       if (changes.type === 'no-change') return;
 
       // Log the entrypoints that were effected
-      internalConfig.logger.info(
+      config.logger.info(
         `Changed: ${Array.from(new Set(fileChanges.map((change) => change[1])))
-          .map((file) => pc.dim(relative(internalConfig.root, file)))
+          .map((file) => pc.dim(relative(config.root, file)))
           .join(', ')}`,
       );
       const rebuiltNames = changes.rebuildGroups
         .flat()
         .map((entry) => {
           return pc.cyan(
-            relative(internalConfig.outDir, getEntrypointOutputFile(entry, '')),
+            relative(config.outDir, getEntrypointOutputFile(entry, '')),
           );
         })
         .join(pc.dim(', '));
 
-      // Get latest config and Rebuild groups with changes
-      internalConfig = await getLatestInternalConfig();
-      internalConfig.server = server;
+      // Rebuild entrypoints on change
       const { output: newOutput } = await rebuild(
-        internalConfig,
+        config,
         // TODO: this excludes new entrypoints, so they're not built until the dev command is restarted
         changes.rebuildGroups,
         changes.cachedOutput,
@@ -108,91 +173,15 @@ export async function createServer(
           server.reloadExtension();
           break;
         case 'html-reload':
-          reloadHtmlPages(changes.rebuildGroups, server, internalConfig);
+          reloadHtmlPages(changes.rebuildGroups, server, config);
           break;
         case 'content-script-reload':
-          reloadContentScripts(changes.changedSteps, internalConfig, server);
+          reloadContentScripts(changes.changedSteps, config, server);
           break;
       }
       consola.success(`Reloaded: ${rebuiltNames}`);
     });
-  });
-
-  return server;
-}
-
-async function getServerInfo(): Promise<ServerInfo> {
-  const { default: getPort, portNumbers } = await import('get-port');
-  const port = await getPort({ port: portNumbers(3000, 3010) });
-  const hostname = 'localhost';
-  const origin = `http://${hostname}:${port}`;
-  const serverConfig: vite.InlineConfig = {
-    server: {
-      origin,
-    },
   };
-
-  return {
-    port,
-    hostname,
-    origin,
-    viteServerConfig: serverConfig,
-  };
-}
-
-async function setupServer(
-  serverInfo: ServerInfo,
-  config: InternalConfig,
-): Promise<WxtDevServer> {
-  const runner = await createExtensionRunner(config);
-
-  const viteServer = await vite.createServer(
-    vite.mergeConfig(serverInfo, await config.vite(config.env)),
-  );
-
-  const start = async () => {
-    await viteServer.listen(server.port);
-    config.logger.success(`Started dev server @ ${serverInfo.origin}`);
-
-    server.currentOutput = await internalBuild(config);
-    await runner.openBrowser(config);
-  };
-
-  const reloadExtension = () => {
-    viteServer.ws.send('wxt:reload-extension');
-  };
-  const reloadPage = (path: string) => {
-    // Can't use Vite's built-in "full-reload" event because it doesn't like our paths, it expects
-    // paths ending in "/index.html"
-    viteServer.ws.send('wxt:reload-page', path);
-  };
-  const reloadContentScript = (
-    contentScript: Omit<Scripting.RegisteredContentScript, 'id'>,
-  ) => {
-    viteServer.ws.send('wxt:reload-content-script', contentScript);
-  };
-
-  const server: WxtDevServer = {
-    ...viteServer,
-    start,
-    currentOutput: {
-      manifest: {
-        manifest_version: 3,
-        name: '',
-        version: '',
-      },
-      publicAssets: [],
-      steps: [],
-    },
-    port: serverInfo.port,
-    hostname: serverInfo.hostname,
-    origin: serverInfo.origin,
-    reloadExtension,
-    reloadPage,
-    reloadContentScript,
-  };
-
-  return server;
 }
 
 /**
@@ -244,10 +233,3 @@ function reloadHtmlPages(
     server.reloadPage(path);
   });
 }
-
-interface ServerInfo {
-  port: number;
-  hostname: string;
-  origin: string;
-  viteServerConfig: vite.InlineConfig;
-}
diff --git a/src/core/utils/building/__tests__/detect-dev-changes.test.ts b/src/core/utils/building/__tests__/detect-dev-changes.test.ts
index ab67a895..4474c167 100644
--- a/src/core/utils/building/__tests__/detect-dev-changes.test.ts
+++ b/src/core/utils/building/__tests__/detect-dev-changes.test.ts
@@ -8,8 +8,8 @@ import {
   fakeManifest,
   fakeOptionsEntrypoint,
   fakePopupEntrypoint,
-  fakeRollupOutputAsset,
-  fakeRollupOutputChunk,
+  fakeOutputAsset,
+  fakeOutputChunk,
 } from '~/core/utils/testing/fake-objects';
 import { BuildOutput, BuildStepOutput } from '~/types';
 
@@ -32,15 +32,11 @@ describe('Detect Dev Changes', () => {
         steps: [
           {
             entrypoints: fakeContentScriptEntrypoint(),
-            chunks: [fakeRollupOutputChunk(), fakeRollupOutputChunk()],
+            chunks: [fakeOutputChunk(), fakeOutputChunk()],
           },
           {
             entrypoints: fakeContentScriptEntrypoint(),
-            chunks: [
-              fakeRollupOutputChunk(),
-              fakeRollupOutputChunk(),
-              fakeRollupOutputChunk(),
-            ],
+            chunks: [fakeOutputChunk(), fakeOutputChunk(), fakeOutputChunk()],
           },
         ],
       };
@@ -57,10 +53,10 @@ describe('Detect Dev Changes', () => {
         'unknown',
         '/root/src/public/image.svg',
       ];
-      const asset1 = fakeRollupOutputAsset({
+      const asset1 = fakeOutputAsset({
         fileName: 'image.svg',
       });
-      const asset2 = fakeRollupOutputAsset({
+      const asset2 = fakeOutputAsset({
         fileName: 'some-other-image.svg',
       });
       const currentOutput: BuildOutput = {
@@ -96,7 +92,7 @@ describe('Detect Dev Changes', () => {
       const step1: BuildStepOutput = {
         entrypoints: contentScript,
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), fakeFile()],
           }),
         ],
@@ -104,7 +100,7 @@ describe('Detect Dev Changes', () => {
       const step2: BuildStepOutput = {
         entrypoints: background,
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), changedPath, fakeFile()],
           }),
         ],
@@ -150,7 +146,7 @@ describe('Detect Dev Changes', () => {
       const step1: BuildStepOutput = {
         entrypoints: [htmlPage1, htmlPage2],
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), changedPath],
           }),
         ],
@@ -158,7 +154,7 @@ describe('Detect Dev Changes', () => {
       const step2: BuildStepOutput = {
         entrypoints: [htmlPage3],
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), fakeFile(), fakeFile()],
           }),
         ],
@@ -203,7 +199,7 @@ describe('Detect Dev Changes', () => {
       const step1: BuildStepOutput = {
         entrypoints: script1,
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), changedPath],
           }),
         ],
@@ -211,7 +207,7 @@ describe('Detect Dev Changes', () => {
       const step2: BuildStepOutput = {
         entrypoints: script2,
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [fakeFile(), fakeFile(), fakeFile()],
           }),
         ],
@@ -219,7 +215,7 @@ describe('Detect Dev Changes', () => {
       const step3: BuildStepOutput = {
         entrypoints: script3,
         chunks: [
-          fakeRollupOutputChunk({
+          fakeOutputChunk({
             moduleIds: [changedPath, fakeFile(), fakeFile()],
           }),
         ],
diff --git a/src/core/utils/building/build-entrypoints.ts b/src/core/utils/building/build-entrypoints.ts
index 4da0fcdd..b5f9d85e 100644
--- a/src/core/utils/building/build-entrypoints.ts
+++ b/src/core/utils/building/build-entrypoints.ts
@@ -1,18 +1,12 @@
-import * as vite from 'vite';
 import {
   BuildOutput,
   BuildStepOutput,
-  Entrypoint,
   EntrypointGroup,
   InternalConfig,
 } from '~/types';
-import * as wxtPlugins from '~/core/vite-plugins';
-import { removeEmptyDirs } from '~/core/utils/fs';
 import { getPublicFiles } from '~/core/utils/fs';
-import { getEntrypointBundlePath } from '~/core/utils/entrypoints';
 import fs from 'fs-extra';
 import { dirname, resolve } from 'path';
-import { getEntrypointGlobals } from '~/core/utils/globals';
 import type { Ora } from 'ora';
 import pc from 'picocolors';
 
@@ -24,155 +18,18 @@ export async function buildEntrypoints(
   const steps: BuildStepOutput[] = [];
   for (let i = 0; i < groups.length; i++) {
     const group = groups[i];
-
-    spinner.text =
-      pc.dim(`[${i + 1}/${groups.length}]`) +
-      ` ${[group]
-        .flat()
-        .map((e) => e.name)
-        .join(pc.dim(', '))}`;
-
-    const step = Array.isArray(group)
-      ? await buildMultipleEntrypoints(group, config)
-      : await buildSingleEntrypoint(group, config);
-    steps.push(step);
+    const groupNames = [group]
+      .flat()
+      .map((e) => e.name)
+      .join(pc.dim(', '));
+    spinner.text = pc.dim(`[${i + 1}/${groups.length}]`) + ` ${groupNames}`;
+    steps.push(await config.builder.build(group));
   }
   const publicAssets = await copyPublicDirectory(config);
 
-  // Remove any empty directories from moving outputs around
-  await removeEmptyDirs(config.outDir);
-
   return { publicAssets, steps };
 }
 
-/**
- * Use Vite's lib mode + IIFE format to bundle the entrypoint to a single file.
- */
-async function buildSingleEntrypoint(
-  entrypoint: Entrypoint,
-  config: InternalConfig,
-): Promise<BuildStepOutput> {
-  // Should this entrypoint be wrapped by the vite-plugins/virtualEntrypoint plugin?
-  const isVirtual = [
-    'background',
-    'content-script',
-    'unlisted-script',
-  ].includes(entrypoint.type);
-  const entry = isVirtual
-    ? `virtual:wxt-${entrypoint.type}?${entrypoint.inputPath}`
-    : entrypoint.inputPath;
-
-  const plugins: NonNullable<vite.UserConfig['plugins']> = [];
-  if (
-    entrypoint.type === 'content-script-style' ||
-    entrypoint.type === 'unlisted-style'
-  ) {
-    plugins.push(wxtPlugins.cssEntrypoints(entrypoint, config));
-  }
-
-  const libMode: vite.UserConfig = {
-    mode: config.mode,
-    plugins,
-    build: {
-      lib: {
-        entry,
-        formats: ['iife'],
-        name: '_',
-        fileName: entrypoint.name,
-      },
-      rollupOptions: {
-        output: {
-          // There's only a single output for this build, so we use the desired bundle path for the
-          // entry output (like "content-scripts/overlay.js")
-          entryFileNames: getEntrypointBundlePath(
-            entrypoint,
-            config.outDir,
-            '.js',
-          ),
-          // Output content script CSS to `content-scripts/`, but all other scripts are written to
-          // `assets/`.
-          assetFileNames: ({ name }) => {
-            if (entrypoint.type === 'content-script' && name?.endsWith('css')) {
-              return `content-scripts/${entrypoint.name}.[ext]`;
-            } else {
-              return `assets/${entrypoint.name}.[ext]`;
-            }
-          },
-        },
-      },
-    },
-    define: {
-      // See https://github.com/aklinker1/vite-plugin-web-extension/issues/96
-      'process.env.NODE_ENV': JSON.stringify(config.mode),
-    },
-  };
-  for (const global of getEntrypointGlobals(config, entrypoint.name)) {
-    libMode.define![global.name] = JSON.stringify(global.value);
-  }
-  const entryConfig = vite.mergeConfig(
-    libMode,
-    await config.vite(config.env),
-  ) as vite.InlineConfig;
-
-  const result = await vite.build(entryConfig);
-  return {
-    entrypoints: entrypoint,
-    chunks: getBuildOutputChunks(result),
-  };
-}
-
-/**
- * Use Vite's multipage build to bundle all the entrypoints in a single step.
- */
-async function buildMultipleEntrypoints(
-  entrypoints: Entrypoint[],
-  config: InternalConfig,
-): Promise<BuildStepOutput> {
-  const multiPage: vite.UserConfig = {
-    mode: config.mode,
-    plugins: [wxtPlugins.multipageMove(entrypoints, config)],
-    build: {
-      rollupOptions: {
-        input: entrypoints.reduce<Record<string, string>>((input, entry) => {
-          input[entry.name] = entry.inputPath;
-          return input;
-        }, {}),
-        output: {
-          // Include a hash to prevent conflicts
-          chunkFileNames: 'chunks/[name]-[hash].js',
-          // Include a hash to prevent conflicts
-          entryFileNames: 'chunks/[name]-[hash].js',
-          // We can't control the "name", so we need a hash to prevent conflicts
-          assetFileNames: 'assets/[name]-[hash].[ext]',
-        },
-      },
-    },
-    define: {},
-  };
-  for (const global of getEntrypointGlobals(config, 'html')) {
-    multiPage.define![global.name] = JSON.stringify(global.value);
-  }
-
-  const entryConfig = vite.mergeConfig(
-    multiPage,
-    await config.vite(config.env),
-  ) as vite.UserConfig;
-
-  const result = await vite.build(entryConfig);
-  return {
-    entrypoints,
-    chunks: getBuildOutputChunks(result),
-  };
-}
-
-function getBuildOutputChunks(
-  result: Awaited<ReturnType<typeof vite.build>>,
-): BuildStepOutput['chunks'] {
-  if ('on' in result) throw Error('wxt does not support vite watch mode.');
-  if (Array.isArray(result)) return result.flatMap(({ output }) => output);
-  return result.output;
-}
-
 async function copyPublicDirectory(
   config: InternalConfig,
 ): Promise<BuildOutput['publicAssets']> {
@@ -189,9 +46,6 @@ async function copyPublicDirectory(
     publicAssets.push({
       type: 'asset',
       fileName: file,
-      name: file,
-      needsCodeReference: false,
-      source: await fs.readFile(srcPath),
     });
   }
 
diff --git a/src/core/utils/building/detect-dev-changes.ts b/src/core/utils/building/detect-dev-changes.ts
index 4dba2998..2981f68e 100644
--- a/src/core/utils/building/detect-dev-changes.ts
+++ b/src/core/utils/building/detect-dev-changes.ts
@@ -1,5 +1,10 @@
-import { BuildOutput, BuildStepOutput, EntrypointGroup } from '~/types';
-import * as vite from 'vite';
+import {
+  BuildOutput,
+  BuildStepOutput,
+  EntrypointGroup,
+  OutputAsset,
+  OutputFile,
+} from '~/types';
 import { every } from '~/core/utils/arrays';
 import { normalizePath } from '~/core/utils/paths';
 
@@ -105,9 +110,7 @@ function findEffectedSteps(
   const changes: DetectedChange[] = [];
   const changedPath = normalizePath(changedFile[1]);
 
-  const isChunkEffected = (
-    chunk: vite.Rollup.OutputChunk | vite.Rollup.OutputAsset,
-  ): boolean =>
+  const isChunkEffected = (chunk: OutputFile): boolean =>
     // If it's an HTML file with the same path, is is effected because HTML files need to be pre-rendered
     // fileName is normalized, relative bundle path
     (chunk.type === 'asset' && changedPath.endsWith(chunk.fileName)) ||
@@ -176,4 +179,4 @@ interface ContentScriptReload extends RebuildChange {
  * directory asset that was changed. It doesn't know what type of change is required yet. Just an
  * intermediate type.
  */
-type DetectedChange = BuildStepOutput | vite.Rollup.OutputAsset;
+type DetectedChange = BuildStepOutput | OutputAsset;
diff --git a/src/core/utils/building/find-entrypoints.ts b/src/core/utils/building/find-entrypoints.ts
index 40412390..4f03efd2 100644
--- a/src/core/utils/building/find-entrypoints.ts
+++ b/src/core/utils/building/find-entrypoints.ts
@@ -22,7 +22,7 @@ import {
   getEntrypointName,
   resolvePerBrowserOption,
 } from '~/core/utils/entrypoints';
-import { VIRTUAL_NOOP_BACKGROUND_MODULE_ID } from '~/core/vite-plugins/noopBackground';
+import { VIRTUAL_NOOP_BACKGROUND_MODULE_ID } from '~/core/utils/constants';
 import { CSS_EXTENSIONS_PATTERN } from '~/core/utils/paths';
 
 /**
diff --git a/src/core/utils/building/generate-wxt-dir.ts b/src/core/utils/building/generate-wxt-dir.ts
index e55fa129..729d51af 100644
--- a/src/core/utils/building/generate-wxt-dir.ts
+++ b/src/core/utils/building/generate-wxt-dir.ts
@@ -156,7 +156,7 @@ async function writeGlobalsDeclarationFile(
   config: InternalConfig,
 ): Promise<string> {
   const filePath = resolve(config.typesDir, 'globals.d.ts');
-  const globals = [...getGlobals(config), ...getEntrypointGlobals(config, '')];
+  const globals = [...getGlobals(config), ...getEntrypointGlobals('')];
   await writeFileIfDifferent(
     filePath,
     [
diff --git a/src/core/utils/building/get-internal-config.ts b/src/core/utils/building/get-internal-config.ts
index e3875590..3663e103 100644
--- a/src/core/utils/building/get-internal-config.ts
+++ b/src/core/utils/building/get-internal-config.ts
@@ -6,14 +6,13 @@ import {
   ConfigEnv,
   UserManifestFn,
   UserManifest,
-  WxtViteConfig,
   ExtensionRunnerConfig,
+  WxtDevServer,
 } from '~/types';
 import path from 'node:path';
-import * as vite from 'vite';
 import { createFsCache } from '~/core/utils/cache';
 import consola, { LogLevels } from 'consola';
-import * as plugins from '~/core/vite-plugins';
+import { craeteViteBuilder } from '~/core/builders/vite';
 import defu from 'defu';
 import { NullablyRequired } from '../types';
 
@@ -27,6 +26,7 @@ import { NullablyRequired } from '../types';
 export async function getInternalConfig(
   inlineConfig: InlineConfig,
   command: 'build' | 'serve',
+  server?: WxtDevServer,
 ): Promise<InternalConfig> {
   // Load user config
 
@@ -93,7 +93,7 @@ export async function getInternalConfig(
     }).map(([key, value]) => [key, path.resolve(root, value)]),
   );
 
-  const finalConfig: InternalConfig = {
+  const finalConfig: Omit<InternalConfig, 'builder'> = {
     browser,
     command,
     debug,
@@ -112,7 +112,6 @@ export async function getInternalConfig(
     runnerConfig,
     srcDir,
     typesDir,
-    vite: () => ({}), // Real value added after this object is initialized.
     wxtDir,
     zip: resolveInternalZipConfig(root, mergedConfig),
     transformManifest(manifest) {
@@ -129,12 +128,19 @@ export async function getInternalConfig(
       includeBrowserPolyfill:
         mergedConfig.experimental?.includeBrowserPolyfill ?? true,
     },
+    server,
   };
 
-  finalConfig.vite = (env) =>
-    resolveInternalViteConfig(env, mergedConfig, finalConfig);
+  const builder = await craeteViteBuilder(
+    inlineConfig,
+    userConfig,
+    finalConfig,
+  );
 
-  return finalConfig;
+  return {
+    ...finalConfig,
+    builder,
+  };
 }
 
 async function resolveManifestConfig(
@@ -159,20 +165,12 @@ function mergeInlineConfig(
   } else if (userConfig.imports == null && inlineConfig.imports == null) {
     imports = undefined;
   } else {
-    imports = vite.mergeConfig(
-      userConfig.imports ?? {},
-      inlineConfig.imports ?? {},
-    );
+    imports = defu(inlineConfig.imports ?? {}, userConfig.imports ?? {});
   }
   const manifest: UserManifestFn = async (env) => {
     const user = await resolveManifestConfig(env, userConfig.manifest);
     const inline = await resolveManifestConfig(env, inlineConfig.manifest);
-    return vite.mergeConfig(user, inline);
-  };
-  const viteConfig = async (env: ConfigEnv): Promise<WxtViteConfig> => {
-    const user = await userConfig.vite?.(env);
-    const inline = await inlineConfig.vite?.(env);
-    return vite.mergeConfig(user ?? {}, inline ?? {});
+    return defu(inline, user);
   };
   const runner: InlineConfig['runner'] = defu(
     inlineConfig.runner ?? {},
@@ -198,7 +196,6 @@ function mergeInlineConfig(
     runner,
     srcDir: inlineConfig.srcDir ?? userConfig.srcDir,
     outDir: inlineConfig.outDir ?? userConfig.outDir,
-    vite: viteConfig,
     zip,
     analysis: {
       enabled: inlineConfig.analysis?.enabled ?? userConfig.analysis?.enabled,
@@ -213,6 +210,7 @@ function mergeInlineConfig(
       ...userConfig.experimental,
       ...inlineConfig.experimental,
     },
+    vite: undefined,
     transformManifest: undefined,
   };
 }
@@ -241,46 +239,3 @@ function resolveInternalZipConfig(
     ],
   };
 }
-
-async function resolveInternalViteConfig(
-  env: ConfigEnv,
-  mergedConfig: InlineConfig,
-  finalConfig: InternalConfig,
-) {
-  const internalVite: vite.InlineConfig =
-    (await mergedConfig.vite?.(env)) ?? {};
-
-  internalVite.root = finalConfig.root;
-  internalVite.configFile = false;
-  internalVite.logLevel = 'warn';
-  internalVite.mode = env.mode;
-
-  internalVite.build ??= {};
-  internalVite.build.outDir = finalConfig.outDir;
-  internalVite.build.emptyOutDir = false;
-
-  internalVite.plugins ??= [];
-  internalVite.plugins.push(plugins.download(finalConfig));
-  internalVite.plugins.push(plugins.devHtmlPrerender(finalConfig));
-  internalVite.plugins.push(plugins.unimport(finalConfig));
-  internalVite.plugins.push(
-    plugins.virtualEntrypoint('background', finalConfig),
-  );
-  internalVite.plugins.push(
-    plugins.virtualEntrypoint('content-script', finalConfig),
-  );
-  internalVite.plugins.push(
-    plugins.virtualEntrypoint('unlisted-script', finalConfig),
-  );
-  internalVite.plugins.push(plugins.devServerGlobals(finalConfig));
-  internalVite.plugins.push(plugins.tsconfigPaths(finalConfig));
-  internalVite.plugins.push(plugins.noopBackground());
-  if (finalConfig.analysis.enabled) {
-    internalVite.plugins.push(plugins.bundleAnalysis());
-  }
-  internalVite.plugins.push(plugins.globals(finalConfig));
-
-  internalVite.plugins.push(plugins.excludeBrowserPolyfill(finalConfig));
-
-  return internalVite;
-}
diff --git a/src/core/utils/building/internal-build.ts b/src/core/utils/building/internal-build.ts
index fa0026fc..b0e230bc 100644
--- a/src/core/utils/building/internal-build.ts
+++ b/src/core/utils/building/internal-build.ts
@@ -1,7 +1,6 @@
 import { findEntrypoints } from './find-entrypoints';
 import { InternalConfig, BuildOutput } from '~/types';
 import pc from 'picocolors';
-import * as vite from 'vite';
 import fs from 'fs-extra';
 import { groupEntrypoints } from './group-entrypoints';
 import { formatDuration } from '~/core/utils/time';
@@ -27,7 +26,7 @@ export async function internalBuild(
   const target = `${config.browser}-mv${config.manifestVersion}`;
   config.logger.info(
     `${verb} ${pc.cyan(target)} for ${pc.cyan(config.mode)} with ${pc.green(
-      `Vite ${vite.version}`,
+      `${config.builder.name} ${config.builder.version}`,
     )}`,
   );
   const startTime = Date.now();
diff --git a/src/core/utils/constants.ts b/src/core/utils/constants.ts
new file mode 100644
index 00000000..117b9627
--- /dev/null
+++ b/src/core/utils/constants.ts
@@ -0,0 +1,5 @@
+/**
+ * Module ID used to build the background in dev mode if the extension doesn't include a background
+ * script/service worker.
+ */
+export const VIRTUAL_NOOP_BACKGROUND_MODULE_ID = 'virtual:user-background';
diff --git a/src/core/utils/fs.ts b/src/core/utils/fs.ts
index 050d2fcb..fe9aada9 100644
--- a/src/core/utils/fs.ts
+++ b/src/core/utils/fs.ts
@@ -2,7 +2,6 @@ import { InternalConfig } from '~/types';
 import fs from 'fs-extra';
 import glob from 'fast-glob';
 import { unnormalizePath } from './paths';
-import path from 'node:path';
 
 /**
  * Only write the contents to a file if it results in a change. This prevents unnecessary file
@@ -36,23 +35,3 @@ export async function getPublicFiles(
   const files = await glob('**/*', { cwd: config.publicDir });
   return files.map(unnormalizePath);
 }
-
-/**
- * Recursively remove all directories that are empty/
- */
-export async function removeEmptyDirs(dir: string): Promise<void> {
-  const files = await fs.readdir(dir);
-  for (const file of files) {
-    const filePath = path.join(dir, file);
-    const stats = await fs.stat(filePath);
-    if (stats.isDirectory()) {
-      await removeEmptyDirs(filePath);
-    }
-  }
-
-  try {
-    await fs.rmdir(dir);
-  } catch {
-    // noop on failure - this means the directory was not empty.
-  }
-}
diff --git a/src/core/utils/globals.ts b/src/core/utils/globals.ts
index 62abd50c..219c83db 100644
--- a/src/core/utils/globals.ts
+++ b/src/core/utils/globals.ts
@@ -1,7 +1,7 @@
 import { InternalConfig } from '~/types';
 
 export function getGlobals(
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): Array<{ name: string; value: any; type: string }> {
   return [
     {
@@ -47,10 +47,7 @@ export function getGlobals(
   ];
 }
 
-export function getEntrypointGlobals(
-  config: InternalConfig,
-  entrypointName: string,
-) {
+export function getEntrypointGlobals(entrypointName: string) {
   return [
     {
       name: surroundInUnderscore('ENTRYPOINT'),
diff --git a/src/core/utils/manifest.ts b/src/core/utils/manifest.ts
index f13d258e..a4485a77 100644
--- a/src/core/utils/manifest.ts
+++ b/src/core/utils/manifest.ts
@@ -44,9 +44,6 @@ export async function writeManifest(
   output.publicAssets.unshift({
     type: 'asset',
     fileName: 'manifest.json',
-    name: 'manifest',
-    needsCodeReference: false,
-    source: str,
   });
 }
 
diff --git a/src/core/utils/network.ts b/src/core/utils/network.ts
index 7e9f57a8..3d48434b 100644
--- a/src/core/utils/network.ts
+++ b/src/core/utils/network.ts
@@ -26,7 +26,7 @@ export async function isOnline(): Promise<boolean> {
  */
 export async function fetchCached(
   url: string,
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): Promise<string> {
   let content: string = '';
 
diff --git a/src/core/utils/testing/fake-objects.ts b/src/core/utils/testing/fake-objects.ts
index 1548a9f6..949d053a 100644
--- a/src/core/utils/testing/fake-objects.ts
+++ b/src/core/utils/testing/fake-objects.ts
@@ -4,7 +4,6 @@
 import { resolve } from 'path';
 import { faker } from '@faker-js/faker';
 import merge from 'lodash.merge';
-import { Rollup } from 'vite';
 import type { Manifest } from 'webextension-polyfill';
 import {
   FsCache,
@@ -15,6 +14,9 @@ import {
   GenericEntrypoint,
   OptionsEntrypoint,
   PopupEntrypoint,
+  OutputChunk,
+  OutputFile,
+  OutputAsset,
 } from '~/types';
 import { mock } from 'vitest-mock-extended';
 
@@ -140,49 +142,19 @@ export const fakeGenericEntrypoint = fakeObjectCreator<GenericEntrypoint>(
   }),
 );
 
-export const fakeRollupOutputChunk = fakeObjectCreator<Rollup.OutputChunk>(
-  () => ({
-    type: 'chunk',
-    code: '',
-    dynamicImports: [],
-    exports: [],
-    facadeModuleId: faker.helpers.arrayElement([null, fakeFile()]),
-    fileName: faker.string.alphanumeric(),
-    implicitlyLoadedBefore: [],
-    importedBindings: {},
-    imports: [],
-    isDynamicEntry: faker.datatype.boolean(),
-    isEntry: faker.datatype.boolean(),
-    isImplicitEntry: faker.datatype.boolean(),
-    map: null,
-    moduleIds: [],
-    modules: {},
-    name: faker.string.alpha(),
-    referencedFiles: [],
-    viteMetadata: {
-      importedAssets: new Set(),
-      importedCss: new Set(),
-    },
-    preliminaryFileName: faker.string.alphanumeric(),
-    sourcemapFileName: null,
-  }),
-);
+export const fakeOutputChunk = fakeObjectCreator<OutputChunk>(() => ({
+  type: 'chunk',
+  fileName: faker.string.alphanumeric(),
+  moduleIds: [],
+}));
 
-export const fakeRollupOutputAsset = fakeObjectCreator<Rollup.OutputAsset>(
-  () => ({
-    type: 'asset',
-    fileName: fakeFileName(),
-    name: faker.string.alpha(),
-    needsCodeReference: faker.datatype.boolean(),
-    source: '',
-  }),
-);
+export const fakeOutputAsset = fakeObjectCreator<OutputAsset>(() => ({
+  type: 'asset',
+  fileName: fakeFileName(),
+}));
 
-export function fakeRollupOutput(): Rollup.OutputAsset | Rollup.OutputChunk {
-  return faker.helpers.arrayElement([
-    fakeRollupOutputAsset(),
-    fakeRollupOutputChunk(),
-  ]);
+export function fakeOutputFile(): OutputFile {
+  return faker.helpers.arrayElement([fakeOutputAsset(), fakeOutputChunk()]);
 }
 
 export const fakeManifest = fakeObjectCreator<Manifest.WebExtensionManifest>(
@@ -228,7 +200,6 @@ export const fakeInternalConfig = fakeObjectCreator<InternalConfig>(() => {
     debug: faker.datatype.boolean(),
     srcDir: fakeDir(),
     typesDir: fakeDir(),
-    vite: () => ({}),
     wxtDir: fakeDir(),
     server: mock<WxtDevServer>(),
     analysis: {
@@ -248,5 +219,6 @@ export const fakeInternalConfig = fakeObjectCreator<InternalConfig>(() => {
     experimental: {
       includeBrowserPolyfill: true,
     },
+    builder: mock(),
   };
 });
diff --git a/src/core/utils/unimport.ts b/src/core/utils/unimport.ts
index 07a2d289..9bcf9083 100644
--- a/src/core/utils/unimport.ts
+++ b/src/core/utils/unimport.ts
@@ -1,9 +1,9 @@
 import { UnimportOptions } from 'unimport';
 import { InternalConfig } from '~/types';
-import { mergeConfig } from 'vite';
+import { defu } from 'defu';
 
 export function getUnimportOptions(
-  config: InternalConfig,
+  config: Omit<InternalConfig, 'builder'>,
 ): Partial<UnimportOptions | false> {
   if (config.imports === false) return false;
 
@@ -23,8 +23,5 @@ export function getUnimportOptions(
     dirs: ['components', 'composables', 'hooks', 'utils'],
   };
 
-  return mergeConfig(
-    defaultOptions,
-    config.imports,
-  ) as Partial<UnimportOptions>;
+  return defu(config.imports, defaultOptions);
 }
diff --git a/src/testing/wxt-vitest-plugin.ts b/src/testing/wxt-vitest-plugin.ts
index cdf3d437..0de35b0d 100644
--- a/src/testing/wxt-vitest-plugin.ts
+++ b/src/testing/wxt-vitest-plugin.ts
@@ -6,7 +6,7 @@ import {
   globals,
   webextensionPolyfillAlias,
   webextensionPolyfillInlineDeps,
-} from '../core/vite-plugins';
+} from '../core/builders/vite/plugins';
 import { getInternalConfig } from '~/core/utils/building';
 import { InlineConfig } from '../types';
 
diff --git a/src/types/external.ts b/src/types/external.ts
index 4ac023a8..b971fcf7 100644
--- a/src/types/external.ts
+++ b/src/types/external.ts
@@ -4,6 +4,7 @@ import { UnimportOptions } from 'unimport';
 import { LogLevel } from 'consola';
 import { ContentScriptContext } from '../client/content-scripts/content-script-context';
 import type { PluginVisualizerOptions } from 'rollup-plugin-visualizer';
+import type { FSWatcher } from 'chokidar';
 
 export interface InlineConfig {
   /**
@@ -89,17 +90,6 @@ export interface InlineConfig {
    * consola
    */
   logger?: Logger;
-  /**
-   * Return custom Vite options from a function. See
-   * <https://vitejs.dev/config/shared-options.html>.
-   *
-   * [`root`](#root), [`configFile`](#configfile), and [`mode`](#mode) should be set in WXT's config
-   * instead of Vite's.
-   *
-   * This is a function because any vite plugins added need to be recreated for each individual
-   * build step, incase they have internal state causing them to fail when reused.
-   */
-  vite?: (env: ConfigEnv) => WxtViteConfig | Promise<WxtViteConfig>;
   /**
    * Customize the `manifest.json` output. Can be an object, promise, or function that returns an
    * object or promise.
@@ -200,10 +190,6 @@ export interface InlineConfig {
    * Add additional paths to the `.wxt/tsconfig.json`. Use this instead of overwriting the `paths`
    * in the root `tsconfig.json` if you want to add new paths.
    *
-   * Passed into Vite's
-   * [`resolve.alias`](https://vitejs.dev/config/shared-options.html#resolve-alias) option and used
-   * to generate the `.wxt/tsconfig.json`.
-   *
    * The key is the import alias and the value is either a relative path to the root directory or an absolute path.
    *
    * @example
@@ -239,43 +225,83 @@ export interface InlineConfig {
   };
 }
 
-export interface WxtInlineViteConfig
-  extends Omit<vite.InlineConfig, 'root' | 'configFile' | 'mode' | 'build'> {
-  build?: Omit<vite.BuildOptions, 'outDir'>;
+// TODO: Extract to @wxt/vite-builder and use module augmentation to include the vite field
+export interface InlineConfig {
+  /**
+   * Return custom Vite options from a function. See
+   * <https://vitejs.dev/config/shared-options.html>.
+   *
+   * [`root`](#root), [`configFile`](#configfile), and [`mode`](#mode) should be set in WXT's config
+   * instead of Vite's.
+   *
+   * This is a function because any vite plugins added need to be recreated for each individual
+   * build step, incase they have internal state causing them to fail when reused.
+   */
+  vite?: (env: ConfigEnv) => WxtViteConfig | Promise<WxtViteConfig>;
 }
 
+// TODO: Move into @wxt/vite-builder
+export type WxtViteConfig = Omit<
+  vite.UserConfig,
+  'root' | 'configFile' | 'mode'
+>;
+
 export interface BuildOutput {
   manifest: Manifest.WebExtensionManifest;
-  publicAssets: vite.Rollup.OutputAsset[];
+  publicAssets: OutputAsset[];
   steps: BuildStepOutput[];
 }
 
+export type OutputFile = OutputChunk | OutputAsset;
+
+export interface OutputChunk {
+  type: 'chunk';
+  /**
+   * Relative, normalized path relative to the output directory.
+   *
+   * Ex: "content-scripts/overlay.js"
+   */
+  fileName: string;
+  /**
+   * Absolute, normalized paths to all dependencies this chunk relies on.
+   */
+  moduleIds: string[];
+}
+
+export interface OutputAsset {
+  type: 'asset';
+  /**
+   * Relative, normalized path relative to the output directory.
+   *
+   * Ex: "icons/16.png"
+   */
+  fileName: string;
+}
+
 export interface BuildStepOutput {
   entrypoints: EntrypointGroup;
-  chunks: (vite.Rollup.OutputChunk | vite.Rollup.OutputAsset)[];
+  chunks: OutputFile[];
 }
 
-export interface WxtDevServer extends vite.ViteDevServer {
-  /**
-   * Ex: `3000`
-   */
-  port: number;
-  /**
-   * Ex: `"localhost"`
-   */
-  hostname: string;
-  /**
-   * Ex: `"http://localhost:3000"`
-   */
-  origin: string;
+export interface WxtDevServer
+  extends Omit<WxtBuilderServer, 'listen'>,
+    ServerInfo {
   /**
    * Stores the current build output of the server.
    */
   currentOutput: BuildOutput;
   /**
-   * Start the server on the first open port.
+   * Start the server.
    */
   start(): Promise<void>;
+  /**
+   * Transform the HTML for dev mode.
+   */
+  transformHtml(
+    url: string,
+    html: string,
+    originalUrl?: string | undefined,
+  ): Promise<string>;
   /**
    * Tell the extension to reload by running `browser.runtime.reload`.
    */
@@ -596,7 +622,73 @@ export interface ExtensionRunnerConfig {
   startUrls?: string[];
 }
 
-export type WxtViteConfig = Omit<
-  vite.UserConfig,
-  'root' | 'configFile' | 'mode'
->;
+export interface WxtBuilder {
+  /**
+   * Name of tool used to build. Ex: "Vite" or "Webpack".
+   */
+  name: string;
+  /**
+   * Version of tool used to build. Ex: "5.0.2"
+   */
+  version: string;
+  /**
+   * Build a single entrypoint group. This is effectively one of the multiple "steps" during the
+   * build process.
+   */
+  build(group: EntrypointGroup): Promise<BuildStepOutput>;
+  /**
+   * Start a dev server at the provided port.
+   */
+  createServer(info: ServerInfo): Promise<WxtBuilderServer>;
+}
+
+export interface WxtBuilderServer {
+  /**
+   * Start the server.
+   */
+  listen(): Promise<void>;
+  /**
+   * Transform the HTML for dev mode.
+   */
+  transformHtml(
+    url: string,
+    html: string,
+    originalUrl?: string | undefined,
+  ): Promise<string>;
+  /**
+   * The web socket server used to communicate with the extension.
+   */
+  ws: {
+    /**
+     * Send a message via the server's websocket, with an optional payload.
+     *
+     * @example
+     * ws.send("wxt:reload-extension");
+     * ws.send("wxt:reload-content-script", { ... });
+     */
+    send(message: string, payload?: any): void;
+    /**
+     * Listen for messages over the server's websocket.
+     */
+    on(message: string, cb: (payload: any) => void): void;
+  };
+  /**
+   * Chokidar file watcher instance.
+   */
+  watcher: FSWatcher;
+}
+
+export interface ServerInfo {
+  /**
+   * Ex: `3000`
+   */
+  port: number;
+  /**
+   * Ex: `"localhost"`
+   */
+  hostname: string;
+  /**
+   * Ex: `"http://localhost:3000"`
+   */
+  origin: string;
+}
diff --git a/src/types/internal.ts b/src/types/internal.ts
index d5caaf07..d1aa8caa 100644
--- a/src/types/internal.ts
+++ b/src/types/internal.ts
@@ -1,4 +1,3 @@
-import * as vite from 'vite';
 import {
   WxtDevServer,
   Logger,
@@ -8,6 +7,7 @@ import {
   ExtensionRunnerConfig,
   ConfigEnv,
   UserConfig,
+  WxtBuilder,
 } from './external';
 import { UnimportOptions } from 'unimport';
 import { ResolvedConfig } from 'c12';
@@ -31,7 +31,6 @@ export interface InternalConfig {
   env: ConfigEnv;
   logger: Logger;
   imports: false | Partial<UnimportOptions>;
-  vite: (env: ConfigEnv) => Promise<vite.InlineConfig> | vite.InlineConfig;
   manifest: UserManifest;
   fsCache: FsCache;
   server?: WxtDevServer;
@@ -56,6 +55,7 @@ export interface InternalConfig {
   experimental: {
     includeBrowserPolyfill: boolean;
   };
+  builder: WxtBuilder;
 }
 
 export interface FsCache {
diff --git a/vitest.workspace.ts b/vitest.workspace.ts
index f5dc69a8..4aa27344 100644
--- a/vitest.workspace.ts
+++ b/vitest.workspace.ts
@@ -1,7 +1,7 @@
 import { defineWorkspace } from 'vitest/config';
 import fs from 'fs-extra';
 import pc from 'picocolors';
-import { Plugin } from 'vite';
+import type { Plugin } from 'vite';
 
 const seed = Math.round(Math.random() * Number.MAX_SAFE_INTEGER);
 console.info('Test seed: ' + pc.cyan(seed));