diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js
index bfac52ccea78a..b606682189fe2 100644
--- a/cli/tsc/99_main_compiler.js
+++ b/cli/tsc/99_main_compiler.js
@@ -147,8 +147,15 @@ delete Object.prototype.__proto__;
     }
   }
 
-  // In the case of the LSP, this will only ever contain the assets.
+  // Cache of asset source files.
   /** @type {Map<string, ts.SourceFile>} */
+  const assetSourceFileCache = new Map();
+
+  // Cache of source files, keyed by specifier.
+  // Stores weak references to ensure we aren't
+  // retaining `ts.SourceFile` objects longer than needed.
+  // This should not include assets, which have a separate cache.
+  /** @type {Map<string, WeakRef<ts.SourceFile>>} */
   const sourceFileCache = new Map();
 
   /** @type {string[]=} */
@@ -576,7 +583,11 @@ delete Object.prototype.__proto__;
       // Needs the original specifier
       specifier = normalizedToOriginalMap.get(specifier) ?? specifier;
 
-      let sourceFile = sourceFileCache.get(specifier);
+      const isAsset = specifier.startsWith(ASSETS_URL_PREFIX);
+
+      let sourceFile = isAsset
+        ? assetSourceFileCache.get(specifier)
+        : sourceFileCache.get(specifier)?.deref();
       if (sourceFile) {
         return sourceFile;
       }
@@ -607,10 +618,12 @@ delete Object.prototype.__proto__;
       );
       sourceFile.moduleName = specifier;
       sourceFile.version = version;
-      if (specifier.startsWith(ASSETS_URL_PREFIX)) {
+      if (isAsset) {
         sourceFile.version = "1";
+        assetSourceFileCache.set(specifier, sourceFile);
+      } else {
+        sourceFileCache.set(specifier, new WeakRef(sourceFile));
       }
-      sourceFileCache.set(specifier, sourceFile);
       scriptVersionCache.set(specifier, version);
       return sourceFile;
     },
@@ -773,9 +786,12 @@ delete Object.prototype.__proto__;
       if (logDebug) {
         debug(`host.getScriptSnapshot("${specifier}")`);
       }
-      let sourceFile = sourceFileCache.get(specifier);
+      const isAsset = specifier.startsWith(ASSETS_URL_PREFIX);
+      let sourceFile = isAsset
+        ? assetSourceFileCache.get(specifier)
+        : sourceFileCache.get(specifier)?.deref();
       if (
-        !specifier.startsWith(ASSETS_URL_PREFIX) &&
+        !isAsset &&
         sourceFile?.version != this.getScriptVersion(specifier)
       ) {
         sourceFileCache.delete(specifier);
@@ -994,13 +1010,11 @@ delete Object.prototype.__proto__;
   function getAssets() {
     /** @type {{ specifier: string; text: string; }[]} */
     const assets = [];
-    for (const sourceFile of sourceFileCache.values()) {
-      if (sourceFile.fileName.startsWith(ASSETS_URL_PREFIX)) {
-        assets.push({
-          specifier: sourceFile.fileName,
-          text: sourceFile.text,
-        });
-      }
+    for (const sourceFile of assetSourceFileCache.values()) {
+      assets.push({
+        specifier: sourceFile.fileName,
+        text: sourceFile.text,
+      });
     }
     return assets;
   }
@@ -1177,8 +1191,9 @@ delete Object.prototype.__proto__;
     "lib.d.ts files have errors",
   );
 
+  assert(buildSpecifier.startsWith(ASSETS_URL_PREFIX));
   // remove this now that we don't need it anymore for warming up tsc
-  sourceFileCache.delete(buildSpecifier);
+  assetSourceFileCache.delete(buildSpecifier);
 
   // exposes the functions that are called by `tsc::exec()` when type
   // checking TypeScript.