feat: lockfiles (#3231)

Use --lock-write=lock.json or --lock-check=lock.json on the command
line.

cli/checksum.rs

@@ -0,0 +1,20 @@
+use ring;
+use std::fmt::Write;
+
+pub fn gen(v: Vec<&[u8]>) -> String {
+ let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
+ for src in v.iter() {
+ ctx.update(src);
+ }
+ let digest = ctx.finish();
+ let mut out = String::new();
+ // TODO There must be a better way to do this...
+ for byte in digest.as_ref() {
+ write!(&mut out, "{:02x}", byte).unwrap();
+ }
+ out
+}
+
+pub fn gen2(s: &str) -> String {
+ gen(vec![s.as_bytes()])
+}

cli/compilers/ts.rs

@@ -18,9 +18,7 @@ use deno::ModuleSpecifier;
 use futures::Future;
 use futures::Stream;
 use regex::Regex;
-use ring;
 use std::collections::HashSet;
-use std::fmt::Write;
 use std::fs;
 use std::io;
 use std::path::PathBuf;
@@ -178,28 +176,14 @@ fn req(
  j.to_string().into_boxed_str().into_boxed_bytes()
 }
 
-fn gen_hash(v: Vec<&[u8]>) -> String {
- let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
- for src in v.iter() {
- ctx.update(src);
- }
- let digest = ctx.finish();
- let mut out = String::new();
- // TODO There must be a better way to do this...
- for byte in digest.as_ref() {
- write!(&mut out, "{:02x}", byte).unwrap();
- }
- out
-}
-
 /// Emit a SHA256 hash based on source code, deno version and TS config.
 /// Used to check if a recompilation for source code is needed.
 pub fn source_code_version_hash(
  source_code: &[u8],
  version: &str,
  config_hash: &[u8],
 ) -> String {
- gen_hash(vec![source_code, version.as_bytes(), config_hash])
+ crate::checksum::gen(vec![source_code, version.as_bytes(), config_hash])
 }
 
 pub struct TsCompiler {

cli/flags.rs

@@ -60,6 +60,9 @@ pub struct DenoFlags {
  pub v8_flags: Option<Vec<String>>,
  // Use tokio::runtime::current_thread
  pub current_thread: bool,
+
+ pub lock: Option<String>,
+ pub lock_write: bool,
 }
 
 static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES:
@@ -131,7 +134,7 @@ pub fn create_cli_app<'a, 'b>() -> App<'a, 'b> {
  .global_settings(&[AppSettings::ColorNever, AppSettings::UnifiedHelpMessage, AppSettings::DisableVersion])
  .settings(&[AppSettings::AllowExternalSubcommands])
  .after_help(ENV_VARIABLES_HELP)
- .long_about("A secure runtime for JavaScript and TypeScript built with V8, Rust, and Tokio.
+ .long_about("A secure JavaScript and TypeScript runtime
 
 Docs: https://deno.land/manual.html
 Modules: https://deno.land/x/
@@ -143,7 +146,7 @@ To run the REPL:
 
 To execute a sandboxed script:
 
- deno https://deno.land/welcome.ts
+ deno https://deno.land/std/examples/welcome.ts
 
 To evaluate code from the command line:
 
@@ -223,6 +226,18 @@ Examples: https://github.com/WICG/import-maps#the-import-map",
  }
  })
  .global(true),
+ ).arg(
+ Arg::with_name("lock")
+ .long("lock")
+ .value_name("FILE")
+ .help("Check the specified lock file")
+ .takes_value(true)
+ .global(true),
+ ).arg(
+ Arg::with_name("lock-write")
+ .long("lock-write")
+ .help("Write lock file. Use with --lock.")
+ .global(true),
  ).arg(
  Arg::with_name("v8-options")
  .long("v8-options")
@@ -634,6 +649,13 @@ pub fn parse_flags(
  }
  }
  }
+ if matches.is_present("lock") {
+ let lockfile = matches.value_of("lock").unwrap();
+ flags.lock = Some(lockfile.to_string());
+ }
+ if matches.is_present("lock-write") {
+ flags.lock_write = true;
+ }
 
  flags = parse_run_args(flags, matches);
  // flags specific to "run" subcommand
@@ -1890,4 +1912,24 @@ mod tests {
  assert_eq!(subcommand, DenoSubcommand::Run);
  assert_eq!(argv, svec!["deno", "script.ts"])
  }
+
+ #[test]
+ fn test_flags_from_vec_38() {
+ let (flags, subcommand, argv) = flags_from_vec(svec![
+ "deno",
+ "--lock-write",
+ "--lock=lock.json",
+ "script.ts"
+ ]);
+ assert_eq!(
+ flags,
+ DenoFlags {
+ lock_write: true,
+ lock: Some("lock.json".to_string()),
+ ..DenoFlags::default()
+ }
+ );
+ assert_eq!(subcommand, DenoSubcommand::Run);
+ assert_eq!(argv, svec!["deno", "script.ts"])
+ }
 }

cli/lib.rs

@@ -17,6 +17,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate url;
 
+mod checksum;
 pub mod colors;
 pub mod compilers;
 pub mod deno_dir;
@@ -32,6 +33,7 @@ mod http_body;
 mod http_util;
 mod import_map;
 mod js;
+mod lockfile;
 pub mod msg;
 pub mod ops;
 pub mod permissions;
@@ -362,6 +364,18 @@ fn run_script(flags: DenoFlags, argv: Vec<String>) {
 
  worker
  .execute_mod_async(&main_module, None, false)
+ .and_then(move |()| {
+ if state.flags.lock_write {
+ if let Some(ref lockfile) = state.lockfile {
+ let g = lockfile.lock().unwrap();
+ g.write()?;
+ } else {
+ eprintln!("--lock flag must be specified when using --lock-write");
+ std::process::exit(11);
+ }
+ }
+ Ok(())
+ })
  .and_then(move |()| {
  js_check(worker.execute("window.dispatchEvent(new Event('load'))"));
  worker.then(move |result| {

cli/lockfile.rs

@@ -0,0 +1,70 @@
+use crate::compilers::CompiledModule;
+use serde_json::json;
+pub use serde_json::Value;
+use std::collections::HashMap;
+use std::io::Result;
+
+pub struct Lockfile {
+ need_read: bool,
+ map: HashMap<String, String>,
+ pub filename: String,
+}
+
+impl Lockfile {
+ pub fn new(filename: String) -> Lockfile {
+ Lockfile {
+ map: HashMap::new(),
+ filename,
+ need_read: true,
+ }
+ }
+
+ pub fn write(&self) -> Result<()> {
+ let j = json!(self.map);
+ let s = serde_json::to_string_pretty(&j).unwrap();
+ let mut f = std::fs::OpenOptions::new()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(&self.filename)?;
+ use std::io::Write;
+ f.write_all(s.as_bytes())?;
+ debug!("lockfile write {}", self.filename);
+ Ok(())
+ }
+
+ pub fn read(&mut self) -> Result<()> {
+ debug!("lockfile read {}", self.filename);
+ let s = std::fs::read_to_string(&self.filename)?;
+ self.map = serde_json::from_str(&s)?;
+ self.need_read = false;
+ Ok(())
+ }
+
+ /// Lazily reads the filename, checks the given module is included.
+ /// Returns Ok(true) if check passed
+ pub fn check(&mut self, m: &CompiledModule) -> Result<bool> {
+ if m.name.starts_with("file:") {
+ return Ok(true);
+ }
+ if self.need_read {
+ self.read()?;
+ }
+ assert!(!self.need_read);
+ Ok(if let Some(lockfile_checksum) = self.map.get(&m.name) {
+ let compiled_checksum = crate::checksum::gen2(&m.code);
+ lockfile_checksum == &compiled_checksum
+ } else {
+ false
+ })
+ }
+
+ // Returns true if module was not already inserted.
+ pub fn insert(&mut self, m: &CompiledModule) -> bool {
+ if m.name.starts_with("file:") {
+ return false;
+ }
+ let checksum = crate::checksum::gen2(&m.code);
+ self.map.insert(m.name.clone(), checksum).is_none()
+ }
+}

cli/state.rs

@@ -9,6 +9,7 @@ use crate::file_fetcher::SourceFileFetcher;
 use crate::flags;
 use crate::global_timer::GlobalTimer;
 use crate::import_map::ImportMap;
+use crate::lockfile::Lockfile;
 use crate::msg;
 use crate::ops::JsonOp;
 use crate::permissions::DenoPermissions;
@@ -88,6 +89,8 @@ pub struct State {
  pub ts_compiler: TsCompiler,
 
  pub include_deno_namespace: bool,
+
+ pub lockfile: Option<Mutex<Lockfile>>,
 }
 
 impl Clone for ThreadSafeState {
@@ -255,6 +258,13 @@ impl ThreadSafeState {
 
  let modules = Arc::new(Mutex::new(deno::Modules::new()));
 
+ // Note: reads lazily from disk on first call to lockfile.check()
+ let lockfile = if let Some(filename) = &flags.lock {
+ Some(Mutex::new(Lockfile::new(filename.to_string())))
+ } else {
+ None
+ };
+
  let state = State {
  main_module,
  modules,
@@ -276,6 +286,7 @@ impl ThreadSafeState {
  js_compiler: JsCompiler {},
  json_compiler: JsonCompiler {},
  include_deno_namespace,
+ lockfile,
  };
 
  Ok(ThreadSafeState(Arc::new(state)))
@@ -285,30 +296,46 @@ impl ThreadSafeState {
  self: &Self,
  module_specifier: &ModuleSpecifier,
  ) -> impl Future<Item = CompiledModule, Error = ErrBox> {
- let state_ = self.clone();
+ let state1 = self.clone();
+ let state2 = self.clone();
 
  self
  .file_fetcher
  .fetch_source_file_async(&module_specifier)
  .and_then(move |out| match out.media_type {
  msg::MediaType::Unknown => {
- state_.js_compiler.compile_async(state_.clone(), &out)
+ state1.js_compiler.compile_async(state1.clone(), &out)
  }
  msg::MediaType::Json => {
- state_.json_compiler.compile_async(state_.clone(), &out)
+ state1.json_compiler.compile_async(state1.clone(), &out)
  }
  msg::MediaType::TypeScript
  | msg::MediaType::TSX
  | msg::MediaType::JSX => {
- state_.ts_compiler.compile_async(state_.clone(), &out)
+ state1.ts_compiler.compile_async(state1.clone(), &out)
  }
  msg::MediaType::JavaScript => {
- if state_.ts_compiler.compile_js {
- state_.ts_compiler.compile_async(state_.clone(), &out)
+ if state1.ts_compiler.compile_js {
+ state1.ts_compiler.compile_async(state1.clone(), &out)
  } else {
- state_.js_compiler.compile_async(state_.clone(), &out)
+ state1.js_compiler.compile_async(state1.clone(), &out)
+ }
+ }
+ })
+ .and_then(move |compiled_module| {
+ if let Some(ref lockfile) = state2.lockfile {
+ let mut g = lockfile.lock().unwrap();
+ if state2.flags.lock_write {
+ g.insert(&compiled_module);
+ } else if !g.check(&compiled_module)? {
+ eprintln!(
+ "Subresource integrety check failed --lock={}\n{}",
+ g.filename, compiled_module.name
+ );
+ std::process::exit(10);
  }
  }
+ Ok(compiled_module)
  })
  }
 

cli/tests/integration_tests.rs

@@ -352,6 +352,34 @@ itest!(_050_more_jsons {
  output: "050_more_jsons.ts.out",
 });
 
+itest!(lock_check_ok {
+ args: "run --lock=lock_check_ok.json http:https://127.0.0.1:4545/cli/tests/003_relative_import.ts",
+ output: "003_relative_import.ts.out",
+ http_server: true,
+});
+
+itest!(lock_check_ok2 {
+ args: "run 019_media_types.ts --lock=lock_check_ok2.json",
+ output: "019_media_types.ts.out",
+ http_server: true,
+});
+
+itest!(lock_check_err {
+ args: "run --lock=lock_check_err.json http:https://127.0.0.1:4545/cli/tests/003_relative_import.ts",
+ output: "lock_check_err.out",
+ check_stderr: true,
+ exit_code: 10,
+ http_server: true,
+});
+
+itest!(lock_check_err2 {
+ args: "run 019_media_types.ts --lock=lock_check_err2.json",
+ output: "lock_check_err2.out",
+ check_stderr: true,
+ exit_code: 10,
+ http_server: true,
+});
+
 itest!(async_error {
  exit_code: 1,
  args: "run --reload async_error.ts",

cli/tests/lock_check_err.json

@@ -0,0 +1,4 @@
+{
+ "http:https://127.0.0.1:4545/cli/tests/subdir/print_hello.ts": "5c93c66125878389f47f4abcac003f4be1276c5223612c26302460d71841e287",
+ "http:https://127.0.0.1:4545/cli/tests/003_relative_import.ts": "bad"
+}

cli/tests/lock_check_err.out

@@ -0,0 +1,2 @@
+[WILDCARD]Subresource integrety check failed --lock=lock_check_err.json
+http:https://127.0.0.1:4545/cli/tests/003_relative_import.ts

cli/tests/lock_check_err2.json

@@ -0,0 +1,9 @@
+{
+ "http:https://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+ "http:https://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": "c320ab0a259760e5c78b9ea840af3cc29697109594a3a5b5cea47128102b3e9d",
+ "http:https://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": "42f66736fea7365ff17d5aa9b9655e8551eb81f360dcfb6b77acdd5c9f699e82",
+ "http:https://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": "54cc82ff3c3b0387df57c7bb8eda4dcd36cbbf499ea483b04ff22c5365d34744",
+ "http:https://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+ "http:https://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+ "http:https://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": "ee0b46f757b5f78681a4eead44820c2349daef7a5903fe3c624f29dbc98772e1"
+}