Import a new city using Overpass, instead of Geofabrik + clipping. a-…

…b-street#523

The returned XML seems to be missing lots of stuff, but it's much faster
and wasn't hard to wire up. I'll look into fixing the query...

abstio/src/download.rs

@@ -9,27 +9,44 @@ use abstutil::prettyprint_usize;
 /// creates an mpsc channel pair and provides the sender. Progress will be described through it.
 pub async fn download_bytes<I: AsRef<str>>(
  url: I,
+ post_body: Option<String>,
  progress: &mut mpsc::Sender<String>,
 ) -> Result<Vec<u8>> {
  let url = url.as_ref();
  info!("Downloading {}", url);
- let mut resp = reqwest::get(url).await.unwrap();
+ let mut resp = if let Some(body) = post_body {
+ reqwest::Client::new()
+ .post(url)
+ .body(body)
+ .send()
+ .await
+ .unwrap()
+ } else {
+ reqwest::get(url).await.unwrap()
+ };
  resp.error_for_status_ref()
  .with_context(|| format!("downloading {}", url))?;
 
  let total_size = resp.content_length().map(|x| x as usize);
  let mut bytes = Vec::new();
  while let Some(chunk) = resp.chunk().await.unwrap() {
- if let Some(n) = total_size {
-  // TODO Throttle?
- if let Err(err) = progress.try_send(format!(
+ // TODO Throttle?
+ let msg = if let Some(n) = total_size {
+ format!(
  "{:.2}% ({} / {} bytes)",
  (bytes.len() as f64) / (n as f64) * 100.0,
  prettyprint_usize(bytes.len()),
  prettyprint_usize(n)
- )) {
- warn!("Couldn't send download progress message: {}", err);
- }
+ )
+ } else {
+ // One example where the HTTP response won't say the response size is the Overpass API
+ format!(
+ "{} bytes (unknown total size)",
+ prettyprint_usize(bytes.len())
+ )
+ };
+ if let Err(err) = progress.try_send(msg) {
+ warn!("Couldn't send download progress message: {}", err);
  }
 
  bytes.write_all(&chunk).unwrap();
@@ -40,10 +57,14 @@ pub async fn download_bytes<I: AsRef<str>>(
 
 /// Download a file from a URL. This must be called with a tokio runtime somewhere. Progress will
 /// be printed to STDOUT.
-pub async fn download_to_file<I1: AsRef<str>, I2: AsRef<str>>(url: I1, path: I2) -> Result<()> {
+pub async fn download_to_file<I1: AsRef<str>, I2: AsRef<str>>(
+ url: I1,
+ post_body: Option<String>,
+ path: I2,
+) -> Result<()> {
  let (mut tx, rx) = futures_channel::mpsc::channel(1000);
  print_download_progress(rx);
- let bytes = download_bytes(url, &mut tx).await?;
+ let bytes = download_bytes(url, post_body, &mut tx).await?;
  let path = path.as_ref();
  std::fs::create_dir_all(std::path::Path::new(path).parent().unwrap())?;
  let mut file = std::fs::File::create(path)?;

geom/src/gps.rs

@@ -3,6 +3,7 @@ use std::fs::File;
 use std::io::{BufRead, BufReader, Write};
 
 use anyhow::Result;
+use geojson::{GeoJson, Value};
 use ordered_float::NotNan;
 use serde::{Deserialize, Serialize};
 
@@ -147,6 +148,34 @@ impl LonLat {
  }
  Some(pts)
  }
+
+ /// Extract polygons from a raw GeoJSON string. For multipolygons, only returns the first
+ /// member.
+ pub fn parse_geojson_polygons(raw: String) -> Result<Vec<Vec<LonLat>>> {
+ let geojson = raw.parse::<GeoJson>()?;
+ let features = match geojson {
+ GeoJson::Feature(feature) => vec![feature],
+ GeoJson::FeatureCollection(feature_collection) => feature_collection.features,
+ _ => anyhow::bail!("Unexpected geojson: {:?}", geojson),
+ };
+ let mut polygons = Vec::new();
+ for mut feature in features {
+ let points = match feature.geometry.take().map(|g| g.value) {
+ Some(Value::MultiPolygon(multi_polygon)) => multi_polygon[0][0].clone(),
+ Some(Value::Polygon(polygon)) => polygon[0].clone(),
+ _ => {
+ anyhow::bail!("Unexpected feature: {:?}", feature);
+ }
+ };
+ polygons.push(
+ points
+ .into_iter()
+ .map(|pt| LonLat::new(pt[0], pt[1]))
+ .collect(),
+ );
+ }
+ Ok(polygons)
+ }
 }
 
 impl fmt::Display for LonLat {

importer/src/bin/geojson_to_osmosis.rs

@@ -1,7 +1,6 @@
 use std::io::{self, Read};
 
 use anyhow::Result;
-use geojson::{GeoJson, Value};
 
 use geom::LonLat;
 
@@ -11,37 +10,13 @@ use geom::LonLat;
 fn main() -> Result<()> {
  let mut buffer = String::new();
  io::stdin().read_to_string(&mut buffer)?;
- let geojson = buffer.parse::<GeoJson>()?;
-
- for (idx, points) in extract_boundaries(geojson)?.into_iter().enumerate() {
+ for (idx, points) in LonLat::parse_geojson_polygons(buffer)?
+ .into_iter()
+ .enumerate()
+ {
  let path = format!("boundary{}.poly", idx);
  LonLat::write_osmosis_polygon(&path, &points)?;
  println!("Wrote {}", path);
  }
  Ok(())
 }
-
-fn extract_boundaries(geojson: GeoJson) -> Result<Vec<Vec<LonLat>>> {
- let features = match geojson {
- GeoJson::Feature(feature) => vec![feature],
- GeoJson::FeatureCollection(feature_collection) => feature_collection.features,
- _ => anyhow::bail!("Unexpected geojson: {:?}", geojson),
- };
- let mut polygons = Vec::new();
- for mut feature in features {
- let points = match feature.geometry.take().map(|g| g.value) {
- Some(Value::MultiPolygon(multi_polygon)) => multi_polygon[0][0].clone(),
- Some(Value::Polygon(polygon)) => polygon[0].clone(),
- _ => {
- anyhow::bail!("Unexpected feature: {:?}", feature);
- }
- };
- polygons.push(
- points
- .into_iter()
- .map(|pt| LonLat::new(pt[0], pt[1]))
- .collect(),
- );
- }
- Ok(polygons)
-}

importer/src/bin/one_step_import.rs

@@ -6,6 +6,7 @@ use anyhow::Result;
 
 use abstio::CityName;
 use abstutil::{must_run_cmd, CmdArgs};
+use geom::LonLat;
 
 /// Import a one-shot A/B Street map in a single command. Takes a GeoJSON file with a boundary as
 /// input. Automatically fetches the OSM data, clips it, and runs the importer.
@@ -14,6 +15,7 @@ async fn main() -> Result<()> {
  let mut args = CmdArgs::new();
  let geojson_path = args.required_free();
  let drive_on_left = args.enabled("--drive_on_left");
+ let use_overpass = args.enabled("--use_overpass");
  args.done();
 
  // Handle running from a binary release or from git. If the latter and the user hasn't built
@@ -31,9 +33,9 @@ async fn main() -> Result<()> {
  println!("Found other executables at {}", bin_dir);
 
  // Convert to a boundary polygon. This tool reads from STDIN.
+ let geojson = abstio::slurp_file(geojson_path)?;
  {
  println!("Converting GeoJSON to Osmosis boundary");
- let geojson = abstio::slurp_file(geojson_path)?;
  let mut cmd = Command::new(format!("{}/geojson_to_osmosis", bin_dir))
  .stdin(Stdio::piped())
  .spawn()?;
@@ -52,44 +54,63 @@ async fn main() -> Result<()> {
  }
  }
 
- // What file should we download?
- let url = {
- println!("Figuring out what Geofabrik file contains your boundary");
- let out = Command::new(format!("{}/pick_geofabrik", bin_dir))
- .arg("boundary0.poly")
- .output()?;
- assert!(out.status.success());
- // pick_geofabrik might output extra lines while downloading the index. Grab the last line.
- let output = String::from_utf8(out.stdout)?;
- output.trim().split('\n').last().unwrap().trim().to_string()
- };
-
- // Name the temporary map based on the Geofabrik region.
  let city = CityName::new("zz", "oneshot");
- let name = abstutil::basename(&url)
- .strip_suffix("-latest.osm")
- .unwrap()
- .to_string();
- let pbf = city.input_path(format!("osm/{}.pbf", abstutil::basename(&url)));
- let osm = city.input_path(format!("osm/{}.osm", name));
- std::fs::create_dir_all(std::path::Path::new(&osm).parent().unwrap())
- .expect("Creating parent dir failed");
+ let name;
+ let osm;
+ if use_overpass {
+ // No easy guess on this without looking at the XML file
+ name = "overpass".to_string();
+ osm = city.input_path(format!("osm/{}.osm", name));
 
- // Download it!
- // TODO This is timing out. Also, really could use progress bars.
- if !abstio::file_exists(&pbf) {
- println!("Downloading {}", url);
- abstio::download_to_file(url, &pbf).await?;
- }
+ let mut polygons = LonLat::parse_geojson_polygons(String::from_utf8(geojson)?)?;
+ let mut filter = "poly:\"".to_string();
+ for pt in polygons.pop().unwrap() {
+ filter.push_str(&format!("{} {} ", pt.y(), pt.x()));
+ }
+ filter.pop();
+ filter.push('"');
+ let query = format!("(\n node({});\n <;\n);\nout meta;\n", filter);
+ abstio::download_to_file("https://overpass-api.de/api/interpreter", Some(query), &osm)
+ .await?;
+ } else {
+ // What file should we download?
+ let url = {
+ println!("Figuring out what Geofabrik file contains your boundary");
+ let out = Command::new(format!("{}/pick_geofabrik", bin_dir))
+ .arg("boundary0.poly")
+ .output()?;
+ assert!(out.status.success());
+ // pick_geofabrik might output extra lines while downloading the index. Grab the last line.
+ let output = String::from_utf8(out.stdout)?;
+ output.trim().split('\n').last().unwrap().trim().to_string()
+ };
+
+ // Name the temporary map based on the Geofabrik region.
+ name = abstutil::basename(&url)
+ .strip_suffix("-latest.osm")
+ .unwrap()
+ .to_string();
+ let pbf = city.input_path(format!("osm/{}.pbf", abstutil::basename(&url)));
+ osm = city.input_path(format!("osm/{}.osm", name));
+ std::fs::create_dir_all(std::path::Path::new(&osm).parent().unwrap())
+ .expect("Creating parent dir failed");
 
- // Clip it
- println!("Clipping osm.pbf file to your boundary");
- must_run_cmd(
- Command::new(format!("{}/clip_osm", bin_dir))
- .arg(format!("--pbf={}", pbf))
- .arg("--clip=boundary0.poly")
- .arg(format!("--out={}", osm)),
- );
+ // Download it!
+ // TODO This is timing out. Also, really could use progress bars.
+ if !abstio::file_exists(&pbf) {
+ println!("Downloading {}", url);
+ abstio::download_to_file(url, None, &pbf).await?;
+ }
+
+ // Clip it
+ println!("Clipping osm.pbf file to your boundary");
+ must_run_cmd(
+ Command::new(format!("{}/clip_osm", bin_dir))
+ .arg(format!("--pbf={}", pbf))
+ .arg("--clip=boundary0.poly")
+ .arg(format!("--out={}", osm)),
+ );
+ }
 
  // Import!
  {

importer/src/bin/pick_geofabrik.rs

@@ -51,7 +51,7 @@ async fn main() -> Result<()> {
 async fn load_remote_geojson(path: String, url: &str) -> Result<GeoJson> {
  if !abstio::file_exists(&path) {
  info!("Downloading {}", url);
- abstio::download_to_file(url, &path).await?;
+ abstio::download_to_file(url, None, &path).await?;
  }
  abstio::maybe_read_json(path, &mut Timer::throwaway())
 }

importer/src/utils.rs

@@ -20,7 +20,7 @@ pub async fn download(config: &ImporterConfiguration, output: String, url: &str)
 
  let tmp = "tmp_output";
  println!("- Missing {}, so downloading {}", output, url);
- abstio::download_to_file(url, tmp).await.unwrap();
+ abstio::download_to_file(url, None, tmp).await.unwrap();
 
  if url.ends_with(".zip") {
  let unzip_to = if output.ends_with('/') {
@@ -70,7 +70,7 @@ pub async fn download_kml(
  std::fs::copy(output.replace(".bin", ".kml"), tmp).unwrap();
  } else {
  println!("- Missing {}, so downloading {}", output, url);
- abstio::download_to_file(url, tmp).await.unwrap();
+ abstio::download_to_file(url, None, tmp).await.unwrap();
  }
 
  println!("- Extracting KML data");

map_gui/src/tools/importer.rs

@@ -62,6 +62,10 @@ impl<A: AppLike + 'static> ImportCity<A> {
  ]),
  Widget::row(vec![
  "Step 5)".text_widget(ctx).centered_vert(),
+ Toggle::choice(ctx, "source", "GeoFabrik", "Overpass", None, true),
+ ]),
+ Widget::row(vec![
+ "Step 6)".text_widget(ctx).centered_vert(),
  ctx.style()
  .btn_solid_primary
  .text("Import the area from your clipboard")
@@ -103,6 +107,9 @@ impl<A: AppLike + 'static> State<A> for ImportCity<A> {
  if self.panel.is_checked("left handed driving") {
  args.push("--drive_on_left".to_string());
  }
+ if !self.panel.is_checked("source") {
+ args.push("--use_overpass".to_string());
+ }
  match grab_geojson_from_clipboard() {
  Ok(()) => Transition::Push(crate::tools::RunCommand::new_state(
  ctx,

map_gui/src/tools/updater.rs

@@ -149,7 +149,7 @@ async fn download_cities(
  warn!("Couldn't send progress: {}", err);
  }
 
- match abstio::download_bytes(&url, &mut inner_progress)
+ match abstio::download_bytes(&url, None, &mut inner_progress)
  .await
  .and_then(|bytes| {
  // TODO Instead of holding everything in memory like this, we could also try to

updater/src/main.rs

@@ -371,7 +371,7 @@ async fn download_file(version: &str, path: &str, dl_from_local: bool) -> Result
  println!("> download {}", url);
  let (mut tx, rx) = futures_channel::mpsc::channel(1000);
  abstio::print_download_progress(rx);
- abstio::download_bytes(url, &mut tx).await
+ abstio::download_bytes(url, None, &mut tx).await
 }
 
 // download() will remove stray files, but leave empty directories around. Since some runtime code