Skip to content

Commit

Permalink
Make bundles fully standalone (denoland#3325)
Browse files Browse the repository at this point in the history 
- Bundles are fully standalone. They now include the shared loader with
  `deno_typescript`.
- Refactor of the loader in `deno_typescript` to perform module
  instantiation in a more
- Change of behaviour when an output file is not specified on the CLI.
  Previously a default name was determined and the bundle written to that
  file, now the bundle will be sent to `stdout`.
- Refactors in the TypeScript compiler to be able to support the concept
  of a request type.  This provides a cleaner abstraction and makes it
  easier to support things like single module transpiles to the userland.
- Remove a "dangerous" circular dependency between `os.ts` and `deno.ts`,
  and define `pid` and `noColor` in a better way.
- Don't bind early to `console` in `repl.ts`.
- Add an integration test for generating a bundle.
  • Loading branch information
@kitsonk @ry
kitsonk authored and ry committed Nov 13, 2019
1 parent ee1b8dc commit 8d03397
Show file tree
Hide file tree
Showing 21 changed files with 336 additions and 480 deletions.
27 changes: 20 additions & 7 deletions cli/compilers/ts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,23 @@ impl CompiledFileMetadata {
}
/// Creates the JSON message send to compiler.ts's onmessage.
fn req(
request_type: msg::CompilerRequestType,
root_names: Vec<String>,
compiler_config: CompilerConfig,
bundle: Option<String>,
out_file: Option<String>,
) -> Buf {
let j = match (compiler_config.path, compiler_config.content) {
(Some(config_path), Some(config_data)) => json!({
"type": request_type as i32,
"rootNames": root_names,
"bundle": bundle,
"outFile": out_file,
"configPath": config_path,
"config": str::from_utf8(&config_data).unwrap(),
}),
_ => json!({
"type": request_type as i32,
"rootNames": root_names,
"bundle": bundle,
"outFile": out_file,
}),
};

Expand Down Expand Up @@ -250,15 +253,20 @@ impl TsCompiler {
self: &Self,
global_state: ThreadSafeGlobalState,
module_name: String,
out_file: String,
out_file: Option<String>,
) -> impl Future<Item = (), Error = ErrBox> {
debug!(
"Invoking the compiler to bundle. module_name: {}",
module_name
);

let root_names = vec![module_name.clone()];
let req_msg = req(root_names, self.config.clone(), Some(out_file));
let req_msg = req(
msg::CompilerRequestType::Bundle,
root_names,
self.config.clone(),
out_file,
);

let worker = TsCompiler::setup_worker(global_state.clone());
let worker_ = worker.clone();
Expand Down Expand Up @@ -360,7 +368,12 @@ impl TsCompiler {
);

let root_names = vec![module_url.to_string()];
let req_msg = req(root_names, self.config.clone(), None);
let req_msg = req(
msg::CompilerRequestType::Compile,
root_names,
self.config.clone(),
None,
);

let worker = TsCompiler::setup_worker(global_state.clone());
let worker_ = worker.clone();
Expand Down Expand Up @@ -709,7 +722,7 @@ mod tests {
.bundle_async(
state.clone(),
module_name,
String::from("$deno$/bundle.js"),
Some(String::from("$deno$/bundle.js")),
)
.then(|result| {
assert!(result.is_ok());
Expand Down
48 changes: 14 additions & 34 deletions cli/flags.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ use clap::Arg;
use clap::ArgMatches;
use clap::Shell;
use clap::SubCommand;
use deno::ModuleSpecifier;
use log::Level;
use std;
use std::str;
Expand Down Expand Up @@ -259,11 +258,16 @@ compiler.",
SubCommand::with_name("bundle")
.about("Bundle module and dependencies into single file")
.long_about(
"Output a single JavaScript file with all dependencies
"Output a single JavaScript file with all dependencies.
If a out_file argument is omitted, the output of the bundle will be sent to
standard out.
Example:
deno bundle https://deno.land/std/examples/colors.ts"
deno bundle https://deno.land/std/examples/colors.ts
deno bundle https://deno.land/std/examples/colors.ts colors.bundle.js"
)
.arg(Arg::with_name("source_file").takes_value(true).required(true))
.arg(Arg::with_name("out_file").takes_value(true).required(false)),
Expand Down Expand Up @@ -793,32 +797,6 @@ pub enum DenoSubcommand {
Version,
}

fn get_default_bundle_filename(source_file: &str) -> String {
let specifier = ModuleSpecifier::resolve_url_or_path(source_file).unwrap();
let path_segments = specifier.as_url().path_segments().unwrap();
let file_name = path_segments.filter(|s| !s.is_empty()).last().unwrap();
let file_stem = file_name.trim_end_matches(".ts").trim_end_matches(".js");
format!("{}.bundle.js", file_stem)
}

#[test]
fn test_get_default_bundle_filename() {
assert_eq!(get_default_bundle_filename("blah.ts"), "blah.bundle.js");
assert_eq!(
get_default_bundle_filename("https://example.com/blah.ts"),
"blah.bundle.js"
);
assert_eq!(get_default_bundle_filename("blah.js"), "blah.bundle.js");
assert_eq!(
get_default_bundle_filename("https://example.com/blah.js"),
"blah.bundle.js"
);
assert_eq!(
get_default_bundle_filename("https://zombo.com/stuff/"),
"stuff.bundle.js"
);
}

pub fn flags_from_vec(
args: Vec<String>,
) -> (DenoFlags, DenoSubcommand, Vec<String>) {
Expand All @@ -835,11 +813,13 @@ pub fn flags_from_vec(
("bundle", Some(bundle_match)) => {
flags.allow_write = true;
let source_file: &str = bundle_match.value_of("source_file").unwrap();
let out_file = bundle_match
.value_of("out_file")
.map(String::from)
.unwrap_or_else(|| get_default_bundle_filename(source_file));
argv.extend(vec![source_file.to_string(), out_file.to_string()]);
let out_file = bundle_match.value_of("out_file").map(String::from);
match out_file {
Some(out_file) => {
argv.extend(vec![source_file.to_string(), out_file.to_string()])
}
_ => argv.extend(vec![source_file.to_string()]),
}
DenoSubcommand::Bundle
}
("completions", Some(completions_match)) => {
Expand Down
109 changes: 84 additions & 25 deletions cli/js/compiler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ enum MediaType {
Unknown = 5
}

// Warning! The values in this enum are duplicated in cli/msg.rs
// Update carefully!
enum CompilerRequestType {
Compile = 0,
Bundle = 1
}

// Startup boilerplate. This is necessary because the compiler has its own
// snapshot. (It would be great if we could remove these things or centralize
// them somewhere else.)
Expand All @@ -44,16 +51,23 @@ window["denoMain"] = denoMain;

const ASSETS = "$asset$";
const OUT_DIR = "$deno$";
const BUNDLE_LOADER = "bundle_loader.js";

/** The format of the work message payload coming from the privileged side */
interface CompilerReq {
type CompilerRequest = {
rootNames: string[];
bundle?: string;
// TODO(ry) add compiler config to this interface.
// options: ts.CompilerOptions;
configPath?: string;
config?: string;
}
} & (
| {
type: CompilerRequestType.Compile;
}
| {
type: CompilerRequestType.Bundle;
outFile?: string;
});

interface ConfigureResponse {
ignoredOptions?: string[];
Expand Down Expand Up @@ -271,7 +285,7 @@ function fetchSourceFiles(
async function processImports(
specifiers: Array<[string, string]>,
referrer = ""
): Promise<void> {
): Promise<SourceFileJson[]> {
if (!specifiers.length) {
return;
}
Expand All @@ -287,6 +301,7 @@ async function processImports(
await processImports(sourceFile.imports(), sourceFile.url);
}
}
return sourceFiles;
}

/** Utility function to turn the number of bytes into a human readable
Expand Down Expand Up @@ -314,16 +329,36 @@ function cache(extension: string, moduleId: string, contents: string): void {
const encoder = new TextEncoder();

/** Given a fileName and the data, emit the file to the file system. */
function emitBundle(fileName: string, data: string): void {
function emitBundle(
rootNames: string[],
fileName: string | undefined,
data: string,
sourceFiles: readonly ts.SourceFile[]
): void {
// For internal purposes, when trying to emit to `$deno$` just no-op
if (fileName.startsWith("$deno$")) {
if (fileName && fileName.startsWith("$deno$")) {
console.warn("skipping emitBundle", fileName);
return;
}
const encodedData = encoder.encode(data);
console.log(`Emitting bundle to "${fileName}"`);
writeFileSync(fileName, encodedData);
console.log(`${humanFileSize(encodedData.length)} emitted.`);
const loader = fetchAsset(BUNDLE_LOADER);
// when outputting to AMD and a single outfile, TypeScript makes up the module
// specifiers which are used to define the modules, and doesn't expose them
// publicly, so we have to try to replicate
const sources = sourceFiles.map(sf => sf.fileName);
const sharedPath = util.commonPath(sources);
rootNames = rootNames.map(id =>
id.replace(sharedPath, "").replace(/\.\w+$/i, "")
);
const instantiate = `instantiate(${JSON.stringify(rootNames)});\n`;
const bundle = `${loader}\n${data}\n${instantiate}`;
if (fileName) {
const encodedData = encoder.encode(bundle);
console.warn(`Emitting bundle to "${fileName}"`);
writeFileSync(fileName, encodedData);
console.warn(`${humanFileSize(encodedData.length)} emitted.`);
} else {
console.log(bundle);
}
}

/** Returns the TypeScript Extension enum for a given media type. */
Expand Down Expand Up @@ -380,17 +415,23 @@ class Host implements ts.CompilerHost {

/** Provides the `ts.HostCompiler` interface for Deno.
*
* @param _rootNames A set of modules that are the ones that should be
* instantiated first. Used when generating a bundle.
* @param _bundle Set to a string value to configure the host to write out a
* bundle instead of caching individual files.
*/
constructor(private _bundle?: string) {
if (this._bundle) {
constructor(
private _requestType: CompilerRequestType,
private _rootNames: string[],
private _outFile?: string
) {
if (this._requestType === CompilerRequestType.Bundle) {
// options we need to change when we are generating a bundle
const bundlerOptions: ts.CompilerOptions = {
module: ts.ModuleKind.AMD,
inlineSourceMap: true,
outDir: undefined,
outFile: `${OUT_DIR}/bundle.js`,
// disabled until we have effective way to modify source maps
sourceMap: false
};
Object.assign(this._options, bundlerOptions);
Expand Down Expand Up @@ -531,10 +572,11 @@ class Host implements ts.CompilerHost {
): void {
util.log("compiler::host.writeFile", fileName);
try {
if (this._bundle) {
emitBundle(this._bundle, data);
assert(sourceFiles != null);
if (this._requestType === CompilerRequestType.Bundle) {
emitBundle(this._rootNames, this._outFile, data, sourceFiles!);
} else {
assert(sourceFiles != null && sourceFiles.length == 1);
assert(sourceFiles.length == 1);
const url = sourceFiles![0].fileName;
const sourceFile = SourceFile.get(url);

Expand Down Expand Up @@ -579,16 +621,29 @@ class Host implements ts.CompilerHost {
// lazy instantiating the compiler web worker
window.compilerMain = function compilerMain(): void {
// workerMain should have already been called since a compiler is a worker.
window.onmessage = async ({ data }: { data: CompilerReq }): Promise<void> => {
const { rootNames, configPath, config, bundle } = data;
util.log(">>> compile start", { rootNames, bundle });
window.onmessage = async ({
data: request
}: {
data: CompilerRequest;
}): Promise<void> => {
const { rootNames, configPath, config } = request;
util.log(">>> compile start", {
rootNames,
type: CompilerRequestType[request.type]
});

// This will recursively analyse all the code for other imports, requesting
// those from the privileged side, populating the in memory cache which
// will be used by the host, before resolving.
await processImports(rootNames.map(rootName => [rootName, rootName]));

const host = new Host(bundle);
const resolvedRootModules = (await processImports(
rootNames.map(rootName => [rootName, rootName])
)).map(info => info.url);

const host = new Host(
request.type,
resolvedRootModules,
request.type === CompilerRequestType.Bundle ? request.outFile : undefined
);
let emitSkipped = true;
let diagnostics: ts.Diagnostic[] | undefined;

Expand Down Expand Up @@ -642,8 +697,9 @@ window.compilerMain = function compilerMain(): void {

// We will only proceed with the emit if there are no diagnostics.
if (diagnostics && diagnostics.length === 0) {
if (bundle) {
console.log(`Bundling "${bundle}"`);
if (request.type === CompilerRequestType.Bundle) {
// warning so it goes to stderr instead of stdout
console.warn(`Bundling "${resolvedRootModules.join(`", "`)}"`);
}
const emitResult = program.emit();
emitSkipped = emitResult.emitSkipped;
Expand All @@ -662,7 +718,10 @@ window.compilerMain = function compilerMain(): void {

postMessage(result);

util.log("<<< compile end", { rootNames, bundle });
util.log("<<< compile end", {
rootNames,
type: CompilerRequestType[request.type]
});

// The compiler isolate exits after a single message.
workerClose();
Expand Down
6 changes: 0 additions & 6 deletions cli/js/deno.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,3 @@ export let pid: number;

/** Reflects the NO_COLOR environment variable: https://no-color.org/ */
export let noColor: boolean;

// TODO(ry) This should not be exposed to Deno.
export function _setGlobals(pid_: number, noColor_: boolean): void {
pid = pid_;
noColor = noColor_;
}

0 comments on commit 8d03397

Please sign in to comment.