Skip to content

Commit

Permalink
migrate to axum (roapi#97)
Browse files Browse the repository at this point in the history
  • Loading branch information
houqp authored Oct 24, 2021
1 parent 3e4bd2e commit 246bc6d
Show file tree
Hide file tree
Showing 15 changed files with 692 additions and 718 deletions.
949 changes: 394 additions & 555 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 0 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ members = [
[patch.crates-io]
datafusion = { git = "https://github.com/houqp/arrow-datafusion.git", rev = "af34cec956c8d67b2520a266e74683d7bdcb3099" }

actix-cors = { git = "https://github.com/houqp/actix-extras.git", rev = "ab3bdb6a5924b6d881d204856199e7539e273d2f" }

deltalake = { git = "https://github.com/houqp/delta-rs.git", rev = "cbf1126e70f8578f192dced7286ea9d63d9f629e" }

[patch."https://github.com/apache/arrow-datafusion"]
Expand Down
2 changes: 1 addition & 1 deletion columnq-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,9 @@ async fn main() -> anyhow::Result<()> {

let app = clap::App::new("Columnq")
.version("0.0.1")
.author("QP Hou")
.about("OLAP the Unix way.")
.setting(clap::AppSettings::SubcommandRequiredElseHelp)
.setting(clap::AppSettings::DisableVersionForSubcommands)
.subcommand(
clap::App::new("sql")
.about("Query tables with SQL")
Expand Down
16 changes: 9 additions & 7 deletions roapi-http/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@ columnq = { path = "../columnq", version = "0", default-features = false }
# for datafusion optimization
snmalloc-rs = { version = "0.2", optional = true }

# all actix dependencies are patched to use git source until actix-web 4.x lands
actix-web = { version = "4.0.0-beta.4", default-features = false }
actix-http = { version = "3.0.0-beta.4", default-features = false }
actix-service = "2.0.0-beta.3"
# see https://github.com/actix/actix-extras/pull/144 for cors tokio 1 upgrade
actix-cors = "*"
# dependencies related to axum
tokio = { version = "1", features = ["rt-multi-thread"] }
hyper = { version = "0", features = ["http1", "server", "stream", "runtime"] }
# axum = "0.2.8"
axum = { git = "https://github.com/tokio-rs/axum.git", rev = "7692baf83728775afbb7ed7f1a178d741ca74c40" }
tower-http = { git = "https://github.com/tower-rs/tower-http.git", branch = "cors", features = ["cors"] }
tower-layer = "0"
tracing = "0"
pin-project = "1"

env_logger = "0"
log = "0"
Expand All @@ -47,7 +50,6 @@ snmalloc = ["snmalloc-rs"]
[dev-dependencies]
actix-rt = "*"
reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"]}
tokio = { version = "1" }

# TODO: uncomment this when we exclude roapi-http from root workspace
# [profile.release]
Expand Down
27 changes: 16 additions & 11 deletions roapi-http/src/api/graphql.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
use actix_web::{web, HttpRequest, HttpResponse};
use std::sync::Arc;

use crate::api::{encode_record_batches, encode_type_from_req, HandlerContext};
use axum::body::Body;
use axum::body::Bytes;
use axum::extract;
use axum::http::header::HeaderMap;
use axum::http::Response;

use crate::api::{encode_record_batches, encode_type_from_hdr, HandlerContext};
use crate::error::ApiErrResp;

pub async fn post(
data: web::Data<HandlerContext>,
req: HttpRequest,
query: web::Bytes,
) -> Result<HttpResponse, ApiErrResp> {
let encode_type = encode_type_from_req(req)?;

let graphq = std::str::from_utf8(&query).map_err(ApiErrResp::read_query)?;
let batches = data.cq.query_graphql(graphq).await?;

state: extract::Extension<Arc<HandlerContext>>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response<Body>, ApiErrResp> {
let ctx = state.0;
let encode_type = encode_type_from_hdr(headers)?;
let graphq = std::str::from_utf8(&body).map_err(ApiErrResp::read_query)?;
let batches = ctx.cq.query_graphql(graphq).await?;
encode_record_batches(encode_type, &batches)
}
32 changes: 25 additions & 7 deletions roapi-http/src/api/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use std::convert::TryFrom;

use actix_web::{http, HttpRequest, HttpResponse};
use axum::body::Body;
use axum::http;
use axum::http::header;
use axum::http::Response;
use columnq::datafusion::arrow;
use columnq::encoding;
use columnq::ColumnQ;
Expand Down Expand Up @@ -32,8 +35,25 @@ impl HandlerContext {
}
}

pub fn encode_type_from_req(req: HttpRequest) -> Result<encoding::ContentType, ApiErrResp> {
match req.headers().get(http::header::ACCEPT) {
#[inline]
pub fn bytes_to_resp(bytes: Vec<u8>, content_type: &'static str) -> Response<Body> {
let mut res = Response::new(Body::from(bytes));
res.headers_mut().insert(
header::CONTENT_TYPE,
header::HeaderValue::from_static(content_type),
);
res
}

#[inline]
pub fn bytes_to_json_resp(bytes: Vec<u8>) -> Response<Body> {
bytes_to_resp(bytes, "application/json")
}

pub fn encode_type_from_hdr(
headers: header::HeaderMap,
) -> Result<encoding::ContentType, ApiErrResp> {
match headers.get(header::ACCEPT) {
None => Ok(encoding::ContentType::Json),
Some(hdr_value) => {
encoding::ContentType::try_from(hdr_value.as_bytes()).map_err(|_| ApiErrResp {
Expand All @@ -48,7 +68,7 @@ pub fn encode_type_from_req(req: HttpRequest) -> Result<encoding::ContentType, A
pub fn encode_record_batches(
content_type: encoding::ContentType,
batches: &[arrow::record_batch::RecordBatch],
) -> Result<HttpResponse, ApiErrResp> {
) -> Result<Response<Body>, ApiErrResp> {
let payload = match content_type {
encoding::ContentType::Json => encoding::json::record_batches_to_bytes(batches)
.map_err(ApiErrResp::json_serialization)?,
Expand All @@ -64,9 +84,7 @@ pub fn encode_record_batches(
.map_err(ApiErrResp::parquet_serialization)?,
};

let mut resp = HttpResponse::Ok();
let builder = resp.content_type(content_type.to_str());
Ok(builder.body(payload))
Ok(bytes_to_resp(payload, content_type.to_str()))
}

pub mod graphql;
Expand Down
35 changes: 15 additions & 20 deletions roapi-http/src/api/rest.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,23 @@
use std::collections::HashMap;
use std::sync::Arc;

use actix_web::{web, HttpRequest, HttpResponse};
use serde_derive::Deserialize;
use axum::body::Body;
use axum::extract;
use axum::http::header::HeaderMap;
use axum::http::Response;

use crate::api::{encode_record_batches, encode_type_from_req, HandlerContext};
use crate::api::HandlerContext;
use crate::api::{encode_record_batches, encode_type_from_hdr};
use crate::error::ApiErrResp;

#[derive(Deserialize)]
pub struct RestTablePath {
table_name: String,
}

pub async fn get_table(
data: web::Data<HandlerContext>,
path: web::Path<RestTablePath>,
req: HttpRequest,
query: web::Query<HashMap<String, String>>,
) -> Result<HttpResponse, ApiErrResp> {
let encode_type = encode_type_from_req(req)?;

let batches = data
.cq
.query_rest_table(&path.table_name, &query.into_inner())
.await?;

state: extract::Extension<Arc<HandlerContext>>,
headers: HeaderMap,
extract::Path(table_name): extract::Path<String>,
extract::Query(params): extract::Query<HashMap<String, String>>,
) -> Result<Response<Body>, ApiErrResp> {
let ctx = &state.0;
let encode_type = encode_type_from_hdr(headers)?;
let batches = ctx.cq.query_rest_table(&table_name, &params).await?;
encode_record_batches(encode_type, &batches)
}
38 changes: 11 additions & 27 deletions roapi-http/src/api/routes.rs
Original file line number Diff line number Diff line change
@@ -1,30 +1,14 @@
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error};

use crate::api;

pub fn register_app_routes<T, B>(app: App<T, B>) -> App<T, B>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.route(
"/api/tables/{table_name}",
web::get().to(api::rest::get_table),
)
.route("/api/sql", web::post().to(api::sql::post))
.route("/api/graphql", web::post().to(api::graphql::post))
.route("/api/schema", web::get().to(api::schema::get))
.route(
"/api/schema/{table_name}",
web::get().to(api::schema::get_by_table_name),
)
use axum::{
routing::{get, post},
Router,
};

pub fn register_app_routes() -> Router {
Router::new()
.route("/api/tables/:table_name", get(api::rest::get_table))
.route("/api/sql", post(api::sql::post))
.route("/api/graphql", post(api::graphql::post))
.route("/api/schema", get(api::schema::schema))
}
55 changes: 24 additions & 31 deletions roapi-http/src/api/schema.rs
Original file line number Diff line number Diff line change
@@ -1,39 +1,32 @@
use std::collections::HashMap;
use std::sync::Arc;

use actix_web::{web, HttpRequest, HttpResponse};
use serde_derive::Deserialize;
use axum::body::Body;
use axum::extract;
use axum::http::Response;

use crate::api::HandlerContext;
use crate::api::{bytes_to_json_resp, HandlerContext};
use crate::error::ApiErrResp;

pub async fn get(
data: web::Data<HandlerContext>,
_req: HttpRequest,
_query: web::Bytes,
) -> Result<HttpResponse, ApiErrResp> {
Ok(HttpResponse::Ok()
.content_type("application/json")
.body(serde_json::to_vec(data.cq.schema_map()).map_err(ApiErrResp::json_serialization)?))
}

#[derive(Deserialize)]
pub struct SchemaTablePath {
table_name: String,
pub async fn schema(
state: extract::Extension<Arc<HandlerContext>>,
) -> Result<Response<Body>, ApiErrResp> {
let ctx = state.0;
let payload =
serde_json::to_vec(ctx.cq.schema_map()).map_err(ApiErrResp::json_serialization)?;
Ok(bytes_to_json_resp(payload))
}

pub async fn get_by_table_name(
data: web::Data<HandlerContext>,
path: web::Path<SchemaTablePath>,
_req: HttpRequest,
_query: web::Query<HashMap<String, String>>,
) -> Result<HttpResponse, ApiErrResp> {
Ok(HttpResponse::Ok().content_type("application/json").body(
serde_json::to_vec(
data.cq
.schema_map()
.get(&path.table_name)
.ok_or_else(|| ApiErrResp::not_found("invalid table name"))?,
)
.map_err(ApiErrResp::json_serialization)?,
))
state: extract::Extension<Arc<HandlerContext>>,
extract::Path(table_name): extract::Path<String>,
) -> Result<Response<Body>, ApiErrResp> {
let ctx = state.0;
let payload = serde_json::to_vec(
ctx.cq
.schema_map()
.get(&table_name)
.ok_or_else(|| ApiErrResp::not_found("invalid table name"))?,
)
.map_err(ApiErrResp::json_serialization)?;
Ok(bytes_to_json_resp(payload))
}
27 changes: 16 additions & 11 deletions roapi-http/src/api/sql.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
use actix_web::{web, HttpRequest, HttpResponse};
use std::sync::Arc;

use crate::api::{encode_record_batches, encode_type_from_req, HandlerContext};
use axum::body::Body;
use axum::body::Bytes;
use axum::extract;
use axum::http::header::HeaderMap;
use axum::http::Response;

use crate::api::{encode_record_batches, encode_type_from_hdr, HandlerContext};
use crate::error::ApiErrResp;

pub async fn post(
data: web::Data<HandlerContext>,
req: HttpRequest,
query: web::Bytes,
) -> Result<HttpResponse, ApiErrResp> {
let encode_type = encode_type_from_req(req)?;

let sql = std::str::from_utf8(&query).map_err(ApiErrResp::read_query)?;
let batches = data.cq.query_sql(sql).await?;

state: extract::Extension<Arc<HandlerContext>>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response<Body>, ApiErrResp> {
let ctx = state.0;
let encode_type = encode_type_from_hdr(headers)?;
let sql = std::str::from_utf8(&body).map_err(ApiErrResp::read_query)?;
let batches = ctx.cq.query_sql(sql).await?;
encode_record_batches(encode_type, &batches)
}
Loading

0 comments on commit 246bc6d

Please sign in to comment.