feat(ext/fetch): support fetching local files (denoland#12545)

Closes denoland#11925
Closes denoland#2150

Co-authored-by: Bert Belder <[email protected]>

cli/tests/unit/fetch_test.ts

@@ -23,7 +23,7 @@ unitTest(
 unitTest({ permissions: { net: true } }, async function fetchProtocolError() {
  await assertRejects(
  async () => {
- await fetch("file:https:///");
+ await fetch("ftp:https://localhost:21/a/file");
  },
  TypeError,
  "not supported",
@@ -1360,3 +1360,59 @@ unitTest(
  client.close();
  },
 );
+
+unitTest(async function fetchFilePerm() {
+ await assertRejects(async () => {
+ await fetch(new URL("../testdata/subdir/json_1.json", import.meta.url));
+ }, Deno.errors.PermissionDenied);
+});
+
+unitTest(async function fetchFilePermDoesNotExist() {
+ await assertRejects(async () => {
+ await fetch(new URL("./bad.json", import.meta.url));
+ }, Deno.errors.PermissionDenied);
+});
+
+unitTest(
+ { permissions: { read: true } },
+ async function fetchFileBadMethod() {
+ await assertRejects(
+ async () => {
+ await fetch(
+ new URL("../testdata/subdir/json_1.json", import.meta.url),
+ {
+ method: "POST",
+ },
+ );
+ },
+ TypeError,
+ "Fetching files only supports the GET method. Received POST.",
+ );
+ },
+);
+
+unitTest(
+ { permissions: { read: true } },
+ async function fetchFileDoesNotExist() {
+ await assertRejects(
+ async () => {
+ await fetch(new URL("./bad.json", import.meta.url));
+ },
+ TypeError,
+ );
+ },
+);
+
+unitTest(
+ { permissions: { read: true } },
+ async function fetchFile() {
+ const res = await fetch(
+ new URL("../testdata/subdir/json_1.json", import.meta.url),
+ );
+ assert(res.ok);
+ const fixture = await Deno.readTextFile(
+ "cli/tests/testdata/subdir/json_1.json",
+ );
+ assertEquals(await res.text(), fixture);
+ },
+);

ext/fetch/fs_fetch_handler.rs

@@ -0,0 +1,52 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use crate::CancelHandle;
+use crate::CancelableResponseFuture;
+use crate::FetchHandler;
+use crate::FetchRequestBodyResource;
+
+use deno_core::error::type_error;
+use deno_core::futures::FutureExt;
+use deno_core::futures::TryFutureExt;
+use deno_core::url::Url;
+use deno_core::CancelFuture;
+use reqwest::StatusCode;
+use std::rc::Rc;
+use tokio_util::io::ReaderStream;
+
+/// An implementation which tries to read file URLs from the file system via
+/// tokio::fs.
+#[derive(Clone)]
+pub struct FsFetchHandler;
+
+impl FetchHandler for FsFetchHandler {
+ fn fetch_file(
+ &mut self,
+ url: Url,
+ ) -> (
+ CancelableResponseFuture,
+ Option<FetchRequestBodyResource>,
+ Option<Rc<CancelHandle>>,
+ ) {
+ let cancel_handle = CancelHandle::new_rc();
+ let response_fut = async move {
+ let path = url.to_file_path()?;
+ let file = tokio::fs::File::open(path).map_err(|_| ()).await?;
+ let stream = ReaderStream::new(file);
+ let body = reqwest::Body::wrap_stream(stream);
+ let response = http::Response::builder()
+ .status(StatusCode::OK)
+ .body(body)
+ .map_err(|_| ())?
+ .into();
+ Ok::<_, ()>(response)
+ }
+ .map_err(move |_| {
+ type_error("NetworkError when attempting to fetch resource.")
+ })
+ .or_cancel(&cancel_handle)
+ .boxed_local();
+
+ (response_fut, None, Some(cancel_handle))
+ }
+}

ext/fetch/lib.rs

@@ -1,5 +1,7 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
+mod fs_fetch_handler;
+
 use data_url::DataUrl;
 use deno_core::error::type_error;
 use deno_core::error::AnyError;
@@ -52,14 +54,21 @@ use tokio_util::io::StreamReader;
 pub use data_url;
 pub use reqwest;
 
-pub fn init<P: FetchPermissions + 'static>(
+pub use fs_fetch_handler::FsFetchHandler;
+
+pub fn init<FP, FH>(
  user_agent: String,
  root_cert_store: Option<RootCertStore>,
  proxy: Option<Proxy>,
  request_builder_hook: Option<fn(RequestBuilder) -> RequestBuilder>,
  unsafely_ignore_certificate_errors: Option<Vec<String>>,
  client_cert_chain_and_key: Option<(String, String)>,
-) -> Extension {
+ file_fetch_handler: FH,
+) -> Extension
+where
+ FP: FetchPermissions + 'static,
+ FH: FetchHandler + 'static,
+{
  Extension::builder()
  .js(include_js_files!(
  prefix "deno:ext/fetch",
@@ -73,13 +82,13 @@ pub fn init<P: FetchPermissions + 'static>(
  "26_fetch.js",
  ))
  .ops(vec![
- ("op_fetch", op_sync(op_fetch::<P>)),
+ ("op_fetch", op_sync(op_fetch::<FP, FH>)),
  ("op_fetch_send", op_async(op_fetch_send)),
  ("op_fetch_request_write", op_async(op_fetch_request_write)),
  ("op_fetch_response_read", op_async(op_fetch_response_read)),
  (
  "op_fetch_custom_client",
- op_sync(op_fetch_custom_client::<P>),
+ op_sync(op_fetch_custom_client::<FP>),
  ),
  ])
  .state(move |state| {
@@ -103,6 +112,7 @@ pub fn init<P: FetchPermissions + 'static>(
  .clone(),
  client_cert_chain_and_key: client_cert_chain_and_key.clone(),
  });
+ state.put::<FH>(file_fetch_handler.clone());
  Ok(())
  })
  .build()
@@ -117,6 +127,45 @@ pub struct HttpClientDefaults {
  pub client_cert_chain_and_key: Option<(String, String)>,
 }
 
+pub type CancelableResponseFuture =
+ Pin<Box<dyn Future<Output = CancelableResponseResult>>>;
+
+pub trait FetchHandler: Clone {
+ // Return the result of the fetch request consisting of a tuple of the
+ // cancelable response result, the optional fetch body resource and the
+ // optional cancel handle.
+ fn fetch_file(
+ &mut self,
+ url: Url,
+ ) -> (
+ CancelableResponseFuture,
+ Option<FetchRequestBodyResource>,
+ Option<Rc<CancelHandle>>,
+ );
+}
+
+/// A default implementation which will error for every request.
+#[derive(Clone)]
+pub struct DefaultFileFetchHandler;
+
+impl FetchHandler for DefaultFileFetchHandler {
+ fn fetch_file(
+ &mut self,
+ _url: Url,
+ ) -> (
+ CancelableResponseFuture,
+ Option<FetchRequestBodyResource>,
+ Option<Rc<CancelHandle>>,
+ ) {
+ let fut = async move {
+ Ok(Err(type_error(
+ "NetworkError when attempting to fetch resource.",
+ )))
+ };
+ (Box::pin(fut), None, None)
+ }
+}
+
 pub trait FetchPermissions {
  fn check_net_url(&mut self, _url: &Url) -> Result<(), AnyError>;
  fn check_read(&mut self, _p: &Path) -> Result<(), AnyError>;
@@ -145,13 +194,14 @@ pub struct FetchReturn {
  cancel_handle_rid: Option<ResourceId>,
 }
 
-pub fn op_fetch<FP>(
+pub fn op_fetch<FP, FH>(
  state: &mut OpState,
  args: FetchArgs,
  data: Option<ZeroCopyBuf>,
 ) -> Result<FetchReturn, AnyError>
 where
  FP: FetchPermissions + 'static,
+ FH: FetchHandler + 'static,
 {
  let client = if let Some(rid) = args.client_rid {
  let r = state.resource_table.get::<HttpClientResource>(rid)?;
@@ -167,6 +217,31 @@ where
  // Check scheme before asking for net permission
  let scheme = url.scheme();
  let (request_rid, request_body_rid, cancel_handle_rid) = match scheme {
+ "file" => {
+ let path = url.to_file_path().map_err(|_| {
+ type_error("NetworkError when attempting to fetch resource.")
+ })?;
+ let permissions = state.borrow_mut::<FP>();
+ permissions.check_read(&path)?;
+
+ if method != Method::GET {
+ return Err(type_error(format!(
+ "Fetching files only supports the GET method. Received {}.",
+ method
+ )));
+ }
+
+ let file_fetch_handler = state.borrow_mut::<FH>();
+ let (request, maybe_request_body, maybe_cancel_handle) =
+ file_fetch_handler.fetch_file(url);
+ let request_rid = state.resource_table.add(FetchRequestResource(request));
+ let maybe_request_body_rid =
+ maybe_request_body.map(|r| state.resource_table.add(r));
+ let maybe_cancel_handle_rid = maybe_cancel_handle
+ .map(|ch| state.resource_table.add(FetchCancelHandle(ch)));
+
+ (request_rid, maybe_request_body_rid, maybe_cancel_handle_rid)
+ }
  "http" | "https" => {
  let permissions = state.borrow_mut::<FP>();
  permissions.check_net_url(&url)?;
@@ -400,7 +475,7 @@ impl Resource for FetchCancelHandle {
  }
 }
 
-struct FetchRequestBodyResource {
+pub struct FetchRequestBodyResource {
  body: AsyncRefCell<mpsc::Sender<std::io::Result<Vec<u8>>>>,
  cancel: CancelHandle,
 }

runtime/build.rs

@@ -121,13 +121,14 @@ mod not_docs {
  deno_url::init(),
  deno_tls::init(),
  deno_web::init(deno_web::BlobStore::default(), Default::default()),
- deno_fetch::init::<Permissions>(
+ deno_fetch::init::<Permissions, deno_fetch::DefaultFileFetchHandler>(
  "".to_owned(),
  None,
  None,
  None,
  None,
  None,
+ deno_fetch::DefaultFileFetchHandler, // No enable_file_fetch
  ),
  deno_websocket::init::<Permissions>("".to_owned(), None, None),
  deno_webstorage::init(None),

runtime/web_worker.rs

@@ -317,13 +317,14 @@ impl WebWorker {
  deno_console::init(),
  deno_url::init(),
  deno_web::init(options.blob_store.clone(), Some(main_module.clone())),
- deno_fetch::init::<Permissions>(
+ deno_fetch::init::<Permissions, deno_fetch::FsFetchHandler>(
  options.user_agent.clone(),
  options.root_cert_store.clone(),
  None,
  None,
  options.unsafely_ignore_certificate_errors.clone(),
  None,
+ deno_fetch::FsFetchHandler,
  ),
  deno_websocket::init::<Permissions>(
  options.user_agent.clone(),

runtime/worker.rs

@@ -101,13 +101,14 @@ impl MainWorker {
  options.blob_store.clone(),
  options.bootstrap.location.clone(),
  ),
- deno_fetch::init::<Permissions>(
+ deno_fetch::init::<Permissions, deno_fetch::FsFetchHandler>(
  options.user_agent.clone(),
  options.root_cert_store.clone(),
  None,
  None,
  options.unsafely_ignore_certificate_errors.clone(),
  None,
+ deno_fetch::FsFetchHandler,
  ),
  deno_websocket::init::<Permissions>(
  options.user_agent.clone(),