Simplify src fetch logic and auto append suffix in cache search (deno…

…land#1322)

src/deno_dir.rs

@@ -1,6 +1,7 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
 use dirs;
 use errors;
+use errors::DenoError;
 use errors::DenoResult;
 use errors::ErrorKind;
 use fs as deno_fs;
@@ -136,38 +137,79 @@ impl DenoDir {
  }
 
  // Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
+ /// Fetch remote source code.
  fn fetch_remote_source(
  self: &Self,
  module_name: &str,
  filename: &str,
- ) -> DenoResult<(String, msg::MediaType)> {
- let p = Path::new(filename);
- // We write a special ".mime" file into the `.deno/deps` directory along side the
- // cached file, containing just the media type.
- let mut media_type_filename = filename.to_string();
- media_type_filename.push_str(".mime");
- let mt = Path::new(&media_type_filename);
-
- let src = if self.reload || !p.exists() {
- eprintln!("Downloading {}", module_name);
- let (source, content_type) = http_util::fetch_sync_string(module_name)?;
- match p.parent() {
- Some(ref parent) => fs::create_dir_all(parent),
- None => Ok(()),
- }?;
- deno_fs::write_file(&p, source.as_bytes(), 0o666)?;
- deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?;
- (source, map_content_type(&p, Some(&content_type)))
- } else {
- let source = fs::read_to_string(&p)?;
- // .mime file might not exists with bundled deps
+ ) -> DenoResult<Option<CodeFetchOutput>> {
+ let extensions = ["", ".ts", ".js"];
+ for ext in extensions.iter() {
+ let filename = [filename, ext].concat();
+ let module_name = [module_name, ext].concat();
+ let p = Path::new(&filename);
+ // We write a special ".mime" file into the `.deno/deps` directory along side the
+ // cached file, containing just the media type.
+ let media_type_filename = [&filename, ".mime"].concat();
+ let mt = Path::new(&media_type_filename);
+ eprint!("Downloading {}...", &module_name); // no newline
+ let maybe_source = http_util::fetch_sync_string(&module_name);
+ if let Ok((source, content_type)) = maybe_source {
+ eprintln!(""); // next line
+ match p.parent() {
+ Some(ref parent) => fs::create_dir_all(parent),
+ None => Ok(()),
+ }?;
+ deno_fs::write_file(&p, source.as_bytes(), 0o666)?;
+ deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?;
+ return Ok(Some(CodeFetchOutput {
+ module_name,
+ filename: filename.clone(), // TODO: no clone after NLL rfc
+ media_type: map_content_type(&p, Some(&content_type)),
+ source_code: source,
+ maybe_output_code: None,
+ maybe_source_map: None,
+ }));
+ } else {
+ eprintln!(" NOT FOUND");
+ }
+ }
+ Ok(None)
+ }
+
+ /// Fetch local or cached source code.
+ fn fetch_local_source(
+ self: &Self,
+ module_name: &str,
+ filename: &str,
+ ) -> DenoResult<Option<CodeFetchOutput>> {
+ let extensions = ["", ".ts", ".js"];
+ for ext in extensions.iter() {
+ let filename = [filename, ext].concat();
+ let module_name = [module_name, ext].concat();
+ let p = Path::new(&filename);
+ if !p.exists() {
+ continue;
+ }
+ let media_type_filename = [&filename, ".mime"].concat();
+ let mt = Path::new(&media_type_filename);
+ let source_code = fs::read_to_string(&p)?;
+ // .mime file might not exists
+ // this is okay for local source: maybe_content_type_str will be None
  let maybe_content_type_string = fs::read_to_string(&mt).ok();
  // Option<String> -> Option<&str>
  let maybe_content_type_str =
  maybe_content_type_string.as_ref().map(String::as_str);
- (source, map_content_type(&p, maybe_content_type_str))
- };
- Ok(src)
+ return Ok(Some(CodeFetchOutput {
+ module_name,
+ filename: filename.clone(), // TODO: no clone after NLL rfc
+ media_type: map_content_type(&p, maybe_content_type_str),
+ source_code,
+ maybe_output_code: None,
+ maybe_source_map: None,
+ }));
+ }
+ Ok(None) // cannot find locally
  }
 
  // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
@@ -177,39 +219,33 @@ impl DenoDir {
  filename: &str,
  ) -> DenoResult<CodeFetchOutput> {
  let is_module_remote = is_remote(module_name);
- let use_extension = |ext| {
- let module_name = format!("{}{}", module_name, ext);
- let filename = format!("{}{}", filename, ext);
- let (source_code, media_type) = if is_module_remote {
- self.fetch_remote_source(&module_name, &filename)?
- } else {
- assert_eq!(
- module_name, filename,
- "if a module isn't remote, it should have the same filename"
- );
- let path = Path::new(&filename);
- (fs::read_to_string(path)?, map_content_type(path, None))
- };
- Ok(CodeFetchOutput {
- module_name: module_name.to_string(),
- filename: filename.to_string(),
- media_type,
- source_code,
- maybe_output_code: None,
- maybe_source_map: None,
- })
- };
- let default_attempt = use_extension("");
- if default_attempt.is_ok() {
- return default_attempt;
+ // We try fetch local. Two cases:
+ // 1. This is a remote module, but no reload provided
+ // 2. This is a local module
+ if !is_module_remote || !self.reload {
+ let maybe_local_source =
+ self.fetch_local_source(&module_name, &filename)?;
+ if let Some(output) = maybe_local_source {
+ return Ok(output);
+ }
+ }
+ // If not remote file, stop here!
+ if !is_module_remote {
+ return Err(DenoError::from(std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ format!("cannot find local file '{}'", filename),
+ )));
  }
- debug!("Trying {}.ts...", module_name);
- let ts_attempt = use_extension(".ts");
- if ts_attempt.is_ok() {
- return ts_attempt;
+ // not cached/local, try remote
+ let maybe_remote_source =
+ self.fetch_remote_source(&module_name, &filename)?;
+ if let Some(output) = maybe_remote_source {
+ return Ok(output);
  }
- debug!("Trying {}.js...", module_name);
- use_extension(".js")
+ return Err(DenoError::from(std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ format!("cannot find remote file '{}'", filename),
+ )));
  }
 
  pub fn code_fetch(
@@ -565,7 +601,72 @@ mod tests {
  }
 
  #[test]
- fn test_fetch_remote_source_1() {
+ fn test_get_source_code() {
+ use tokio_util;
+ // http_util::fetch_sync_string requires tokio
+ tokio_util::init(|| {
+ let (temp_dir, deno_dir) = test_setup();
+ let module_name = "http:https://localhost:4545/tests/subdir/mod2.ts";
+ let filename = deno_fs::normalize_path(
+ deno_dir
+ .deps_http
+ .join("localhost_PORT4545/tests/subdir/mod2.ts")
+ .as_ref(),
+ );
+ let mime_file_name = format!("{}.mime", &filename);
+
+ let result = deno_dir.get_source_code(module_name, &filename);
+ assert!(result.is_ok());
+ let r = result.unwrap();
+ assert_eq!(
+ &(r.source_code),
+ "export { printHello } from \"./print_hello.ts\";\n"
+ );
+ assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
+ assert_eq!(
+ fs::read_to_string(&mime_file_name).unwrap(),
+ "application/typescript"
+ );
+
+ // Modify .mime
+ let _ = fs::write(&mime_file_name, "text/javascript");
+ let result2 = deno_dir.get_source_code(module_name, &filename);
+ assert!(result2.is_ok());
+ let r2 = result2.unwrap();
+ assert_eq!(
+ &(r2.source_code),
+ "export { printHello } from \"./print_hello.ts\";\n"
+ );
+ // If get_source_code does not call remote, this should be JavaScript
+ // as we modified before! (we do not overwrite .mime due to no http fetch)
+ assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
+ assert_eq!(
+ fs::read_to_string(&mime_file_name).unwrap(),
+ "text/javascript"
+ );
+
+ // Force self.reload
+ let deno_dir = DenoDir::new(true, Some(temp_dir.path().to_path_buf()))
+ .expect("setup fail");
+ let result3 = deno_dir.get_source_code(module_name, &filename);
+ assert!(result3.is_ok());
+ let r3 = result3.unwrap();
+ assert_eq!(
+ &(r3.source_code),
+ "export { printHello } from \"./print_hello.ts\";\n"
+ );
+ // Now the .mime file should be overwritten back to TypeScript!
+ // (due to http fetch)
+ assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
+ assert_eq!(
+ fs::read_to_string(&mime_file_name).unwrap(),
+ "application/typescript"
+ );
+ });
+ }
+
+ #[test]
+ fn test_fetch_source_1() {
  use tokio_util;
  // http_util::fetch_sync_string requires tokio
  tokio_util::init(|| {
@@ -582,24 +683,24 @@ mod tests {
 
  let result = deno_dir.fetch_remote_source(module_name, &filename);
  assert!(result.is_ok());
- let r = result.unwrap();
- assert_eq!(&(r.0), "export const loaded = true;\n");
- assert_eq!(&(r.1), &msg::MediaType::TypeScript);
+ let r = result.unwrap().unwrap();
+ assert_eq!(&(r.source_code), "export const loaded = true;\n");
+ assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
  assert_eq!(fs::read_to_string(&mime_file_name).unwrap(), "video/mp2t");
 
- // Modify .mime, make sure still read from local
+ // Modify .mime, make sure read from local
  let _ = fs::write(&mime_file_name, "text/javascript");
- let result2 = deno_dir.fetch_remote_source(module_name, &filename);
+ let result2 = deno_dir.fetch_local_source(module_name, &filename);
  assert!(result2.is_ok());
- let r2 = result2.unwrap();
- assert_eq!(&(r2.0), "export const loaded = true;\n");
+ let r2 = result2.unwrap().unwrap();
+ assert_eq!(&(r2.source_code), "export const loaded = true;\n");
  // Not MediaType::TypeScript due to .mime modification
- assert_eq!(&(r2.1), &msg::MediaType::JavaScript);
+ assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
  });
  }
 
  #[test]
- fn test_fetch_remote_source_2() {
+ fn test_fetch_source_2() {
  // only local, no http_util::fetch_sync_string called
  let (_temp_dir, deno_dir) = test_setup();
  let cwd = std::env::current_dir().unwrap();
@@ -608,11 +709,11 @@ mod tests {
  let filename =
  format!("{}/tests/subdir/mt_text_typescript.t1.ts", &cwd_string);
 
- let result = deno_dir.fetch_remote_source(module_name, &filename);
+ let result = deno_dir.fetch_local_source(module_name, &filename);
  assert!(result.is_ok());
- let r = result.unwrap();
- assert_eq!(&(r.0), "export const loaded = true;\n");
- assert_eq!(&(r.1), &msg::MediaType::TypeScript);
+ let r = result.unwrap().unwrap();
+ assert_eq!(&(r.source_code), "export const loaded = true;\n");
+ assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
  }
 
  #[test]