Add stor family of commands (nushell#11170)

# Description

This PR adds the `stor` family of commands. These commands are meant to
create, open, insert, update, delete, reset data in an in-memory sqlite
database. This is really an experiment to see how creatively we can use
an in-memory database.

```
Usage:
  > stor

Subcommands:
  stor create - Create a table in the in-memory sqlite database
  stor delete - Delete a table or specified rows in the in-memory sqlite database
  stor export - Export the in-memory sqlite database to a sqlite database file
  stor import - Import a sqlite database file into the in-memory sqlite database
  stor insert - Insert information into a specified table in the in-memory sqlite database
  stor open - Opens the in-memory sqlite database
  stor reset - Reset the in-memory database by dropping all tables
  stor update - Update information in a specified table in the in-memory sqlite database

Flags:
  -h, --help - Display the help message for this command

Input/output types:
  ╭─#─┬──input──┬─output─╮
  │ 0 │ nothing │ string │
  ╰───┴─────────┴────────╯
```

### Examples

## stor create
```nushell
❯ stor create --table-name nudb --columns {bool1: bool, int1: int, float1: float, str1: str, datetime1: datetime}
╭──────┬────────────────╮
│ nudb │ [list 0 items] │
╰──────┴────────────────╯
```
## stor insert
```nushell
❯ stor insert --table-name nudb --data-record {bool1: true, int1: 2, float1: 1.1, str1: fdncred, datetime1: 2023-04-17} 
╭──────┬───────────────╮
│ nudb │ [table 1 row] │
╰──────┴───────────────╯
```
## stor open
```nushell
❯ stor open | table -e 
╭──────┬────────────────────────────────────────────────────────────────────╮
│      │ ╭─#─┬id─┬bool1┬int1┬float1┬──str1───┬─────────datetime1──────────╮ │
│ nudb │ │ 0 │ 1 │   1 │  2 │ 1.10 │ fdncred │ 2023-04-17 00:00:00 +00:00 │ │
│      │ ╰───┴───┴─────┴────┴──────┴─────────┴────────────────────────────╯ │
╰──────┴────────────────────────────────────────────────────────────────────╯
```
## stor update
```nushell
❯ stor update --table-name nudb --update-record {str1: toby datetime1: 2021-04-17} --where-clause "bool1 = 1"
╭──────┬───────────────╮
│ nudb │ [table 1 row] │
╰──────┴───────────────╯
❯ stor open | table -e
╭──────┬─────────────────────────────────────────────────────────────────╮
│      │ ╭─#─┬id─┬bool1┬int1┬float1┬─str1─┬─────────datetime1──────────╮ │
│ nudb │ │ 0 │ 1 │   1 │  2 │ 1.10 │ toby │ 2021-04-17 00:00:00 +00:00 │ │
│      │ ╰───┴───┴─────┴────┴──────┴──────┴────────────────────────────╯ │
╰──────┴─────────────────────────────────────────────────────────────────╯
```
## insert another row
```nushell
❯ stor insert --table-name nudb --data-record {bool1: true, int1: 5, float1: 1.1, str1: fdncred, datetime1: 2023-04-17} 
╭──────┬────────────────╮
│ nudb │ [table 2 rows] │
╰──────┴────────────────╯
❯ stor open | table -e
╭──────┬────────────────────────────────────────────────────────────────────╮
│      │ ╭─#─┬id─┬bool1┬int1┬float1┬──str1───┬─────────datetime1──────────╮ │
│ nudb │ │ 0 │ 1 │   1 │  2 │ 1.10 │ toby    │ 2021-04-17 00:00:00 +00:00 │ │
│      │ │ 1 │ 2 │   1 │  5 │ 1.10 │ fdncred │ 2023-04-17 00:00:00 +00:00 │ │
│      │ ╰───┴───┴─────┴────┴──────┴─────────┴────────────────────────────╯ │
╰──────┴────────────────────────────────────────────────────────────────────╯
```
## stor delete (specific row(s))
```nushell
❯ stor delete --table-name nudb --where-clause "int1 == 5"
╭──────┬───────────────╮
│ nudb │ [table 1 row] │
╰──────┴───────────────╯
```
## insert multiple tables
```nushell
❯ stor create --table-name nudb1 --columns {bool1: bool, int1: int, float1: float, str1: str, datetime1: datetime}
╭───────┬────────────────╮
│ nudb  │ [table 1 row]  │
│ nudb1 │ [list 0 items] │
╰───────┴────────────────╯
❯ stor insert --table-name nudb1 --data-record {bool1: true, int1: 2, float1: 1.1, str1: fdncred, datetime1: 2023-04-17}
╭───────┬───────────────╮
│ nudb  │ [table 1 row] │
│ nudb1 │ [table 1 row] │
╰───────┴───────────────╯
❯ stor create --table-name nudb2 --columns {bool1: bool, int1: int, float1: float, str1: str, datetime1: datetime}
╭───────┬────────────────╮
│ nudb  │ [table 1 row]  │
│ nudb1 │ [table 1 row]  │
│ nudb2 │ [list 0 items] │
╰───────┴────────────────╯
❯ stor insert --table-name nudb2 --data-record {bool1: true, int1: 2, float1: 1.1, str1: fdncred, datetime1: 2023-04-17}
╭───────┬───────────────╮
│ nudb  │ [table 1 row] │
│ nudb1 │ [table 1 row] │
│ nudb2 │ [table 1 row] │
╰───────┴───────────────╯
```
## stor delete (specific table)
```nushell
❯ stor delete --table-name nudb1
╭───────┬───────────────╮
│ nudb  │ [table 1 row] │
│ nudb2 │ [table 1 row] │
╰───────┴───────────────╯
```
## stor reset (all tables are deleted)
```nushell
❯ stor reset
```
## stor export
```nushell
❯ stor export --file-name nudb.sqlite3
╭──────┬───────────────╮
│ nudb │ [table 1 row] │
╰──────┴───────────────╯
❯ open nudb.sqlite3 | table -e
╭──────┬────────────────────────────────────────────────────────────────────╮
│      │ ╭─#─┬id─┬bool1┬int1┬float1┬──str1───┬─────────datetime1──────────╮ │
│ nudb │ │ 0 │ 1 │   1 │  5 │ 1.10 │ fdncred │ 2023-04-17 00:00:00 +00:00 │ │
│      │ ╰───┴───┴─────┴────┴──────┴─────────┴────────────────────────────╯ │
╰──────┴────────────────────────────────────────────────────────────────────╯
❯ open nudb.sqlite3 | schema | table -e
╭────────┬─────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│        │ ╭──────┬──────────────────────────────────────────────────────────────────────────────────────────────────╮ │
│ tables │ │      │ ╭───────────────┬──────────────────────────────────────────────────────────────────────────────╮ │ │
│        │ │ nudb │ │               │ ╭─#─┬─cid─┬───name────┬─────type─────┬─notnull─┬───────default────────┬─pk─╮ │ │ │
│        │ │      │ │ columns       │ │ 0 │ 0   │ id        │ INTEGER      │ 1       │                      │ 1  │ │ │ │
│        │ │      │ │               │ │ 1 │ 1   │ bool1     │ BOOLEAN      │ 0       │                      │ 0  │ │ │ │
│        │ │      │ │               │ │ 2 │ 2   │ int1      │ INTEGER      │ 0       │                      │ 0  │ │ │ │
│        │ │      │ │               │ │ 3 │ 3   │ float1    │ REAL         │ 0       │                      │ 0  │ │ │ │
│        │ │      │ │               │ │ 4 │ 4   │ str1      │ VARCHAR(255) │ 0       │                      │ 0  │ │ │ │
│        │ │      │ │               │ │ 5 │ 5   │ datetime1 │ DATETIME     │ 0       │ STRFTIME('%Y-%m-%d   │ 0  │ │ │ │
│        │ │      │ │               │ │   │     │           │              │         │ %H:%M:%f', 'NOW')    │    │ │ │ │
│        │ │      │ │               │ ╰─#─┴─cid─┴───name────┴─────type─────┴─notnull─┴───────default────────┴─pk─╯ │ │ │
│        │ │      │ │ constraints   │ [list 0 items]                                                               │ │ │
│        │ │      │ │ foreign_keys  │ [list 0 items]                                                               │ │ │
│        │ │      │ │ indexes       │ [list 0 items]                                                               │ │ │
│        │ │      │ ╰───────────────┴──────────────────────────────────────────────────────────────────────────────╯ │ │
│        │ ╰──────┴──────────────────────────────────────────────────────────────────────────────────────────────────╯ │
╰────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
```
## Using with `query db`
```nushell
❯ stor open | query db "select * from nudb"
╭─#─┬id─┬bool1┬int1┬float1┬──str1───┬─────────datetime1──────────╮
│ 0 │ 1 │   1 │  5 │ 1.10 │ fdncred │ 2023-04-17 00:00:00 +00:00 │
╰───┴───┴─────┴────┴──────┴─────────┴────────────────────────────╯
```
## stor import
```nushell
❯ stor open
# note, nothing is returned. there is nothing in memory, atm.
❯ stor import --file-name nudb.sqlite3
╭──────┬───────────────╮
│ nudb │ [table 1 row] │
╰──────┴───────────────╯
❯ stor open | table -e 
╭──────┬────────────────────────────────────────────────────────────────────╮
│      │ ╭─#─┬id─┬bool1┬int1┬float1┬──str1───┬─────────datetime1──────────╮ │
│ nudb │ │ 0 │ 1 │   1 │  5 │ 1.10 │ fdncred │ 2023-04-17 00:00:00 +00:00 │ │
│      │ ╰───┴───┴─────┴────┴──────┴─────────┴────────────────────────────╯ │
╰──────┴────────────────────────────────────────────────────────────────────╯
```

TODO:
- [x] `stor export` - Export a fully formed sqlite db file. 
- [x] `stor import` - Imports a specified sqlite db file.
- [x] Perhaps feature-gate it with the sqlite feature
- [x] Update `query db` to work with the in-memory database
- [x] Remove `open --in-memory`

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use std testing; testing run-tests --path
crates/nu-std"` to run the tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

crates/nu-command/Cargo.toml

@@ -73,7 +73,7 @@ rand = "0.8"
 rayon = "1.8"
 regex = "1.9.5"
 roxmltree = "0.18"
-rusqlite = { version = "0.29", features = ["bundled"], optional = true }
+rusqlite = { version = "0.29", features = ["bundled", "backup"], optional = true }
 same-file = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

crates/nu-command/src/database/mod.rs

@@ -5,7 +5,7 @@ use commands::add_commands_decls;
 
 pub use values::{
  convert_sqlite_row_to_nu_value, convert_sqlite_value_to_nu_value, open_connection_in_memory,
- SQLiteDatabase,
+ open_connection_in_memory_custom, SQLiteDatabase, MEMORY_DB,
 };
 
 use nu_protocol::engine::StateWorkingSet;

crates/nu-command/src/database/values/mod.rs

@@ -3,5 +3,5 @@ pub mod sqlite;
 
 pub use sqlite::{
  convert_sqlite_row_to_nu_value, convert_sqlite_value_to_nu_value, open_connection_in_memory,
- SQLiteDatabase,
+ open_connection_in_memory_custom, SQLiteDatabase, MEMORY_DB,
 };

crates/nu-command/src/database/values/sqlite.rs

@@ -2,9 +2,10 @@ use super::definitions::{
  db_column::DbColumn, db_constraint::DbConstraint, db_foreignkey::DbForeignKey,
  db_index::DbIndex, db_table::DbTable,
 };
-
 use nu_protocol::{CustomValue, PipelineData, Record, ShellError, Span, Spanned, Value};
-use rusqlite::{types::ValueRef, Connection, Row};
+use rusqlite::{
+ types::ValueRef, Connection, DatabaseName, Error as SqliteError, OpenFlags, Row, Statement,
+};
 use serde::{Deserialize, Serialize};
 use std::{
  fs::File,
@@ -14,8 +15,9 @@ use std::{
 };
 
 const SQLITE_MAGIC_BYTES: &[u8] = "SQLite format 3\0".as_bytes();
+pub const MEMORY_DB: &str = "file:memdb1?mode=memory&cache=shared";
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SQLiteDatabase {
  // I considered storing a SQLite connection here, but decided against it because
  // 1) YAGNI, 2) it's not obvious how cloning a connection could work, 3) state
@@ -85,13 +87,14 @@ impl SQLiteDatabase {
  }
 
  pub fn into_value(self, span: Span) -> Value {
- Value::custom_value(Box::new(self), span)
+ let db = Box::new(self);
+ Value::custom_value(db, span)
  }
 
  pub fn query(&self, sql: &Spanned<String>, call_span: Span) -> Result<Value, ShellError> {
- let db = open_sqlite_db(&self.path, call_span)?;
+ let conn = open_sqlite_db(&self.path, call_span)?;
 
- let stream = run_sql_query(db, sql, self.ctrlc.clone()).map_err(|e| {
+ let stream = run_sql_query(conn, sql, self.ctrlc.clone()).map_err(|e| {
  ShellError::GenericError(
  "Failed to query SQLite database".into(),
  e.to_string(),
@@ -104,11 +107,23 @@ impl SQLiteDatabase {
  Ok(stream)
  }
 
- pub fn open_connection(&self) -> Result<Connection, rusqlite::Error> {
- Connection::open(&self.path)
+ pub fn open_connection(&self) -> Result<Connection, ShellError> {
+ if self.path == PathBuf::from(MEMORY_DB) {
+ open_connection_in_memory_custom()
+ } else {
+ Connection::open(&self.path).map_err(|e| {
+ ShellError::GenericError(
+ "Failed to open SQLite database from open_connection".into(),
+ e.to_string(),
+ None,
+ None,
+ Vec::new(),
+ )
+ })
+ }
  }
 
- pub fn get_tables(&self, conn: &Connection) -> Result<Vec<DbTable>, rusqlite::Error> {
+ pub fn get_tables(&self, conn: &Connection) -> Result<Vec<DbTable>, SqliteError> {
  let mut table_names =
  conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?;
  let rows = table_names.query_map([], |row| row.get(0))?;
@@ -128,7 +143,59 @@ impl SQLiteDatabase {
  Ok(tables.into_iter().collect())
  }
 
- fn get_column_info(&self, row: &Row) -> Result<DbColumn, rusqlite::Error> {
+ pub fn drop_all_tables(&self, conn: &Connection) -> Result<(), SqliteError> {
+ let tables = self.get_tables(conn)?;
+
+ for table in tables {
+ conn.execute(&format!("DROP TABLE {}", table.name), [])?;
+ }
+
+ Ok(())
+ }
+
+ pub fn export_in_memory_database_to_file(
+ &self,
+ conn: &Connection,
+ filename: String,
+ ) -> Result<(), SqliteError> {
+ //vacuum main into 'c:\\temp\\foo.db'
+ conn.execute(&format!("vacuum main into '{}'", filename), [])?;
+
+ Ok(())
+ }
+
+ pub fn backup_database_to_file(
+ &self,
+ conn: &Connection,
+ filename: String,
+ ) -> Result<(), SqliteError> {
+ conn.backup(DatabaseName::Main, Path::new(&filename), None)?;
+ Ok(())
+ }
+
+ pub fn restore_database_from_file(
+ &self,
+ conn: &mut Connection,
+ filename: String,
+ ) -> Result<(), SqliteError> {
+ conn.restore(
+ DatabaseName::Main,
+ Path::new(&filename),
+ Some(|p: rusqlite::backup::Progress| {
+ let percent = if p.pagecount == 0 {
+ 100
+ } else {
+ (p.pagecount - p.remaining) * 100 / p.pagecount
+ };
+ if percent % 10 == 0 {
+ log::trace!("Restoring: {} %", percent);
+ }
+ }),
+ )?;
+ Ok(())
+ }
+
+ fn get_column_info(&self, row: &Row) -> Result<DbColumn, SqliteError> {
  let dbc = DbColumn {
  cid: row.get("cid")?,
  name: row.get("name")?,
@@ -144,7 +211,7 @@ impl SQLiteDatabase {
  &self,
  conn: &Connection,
  table: &DbTable,
- ) -> Result<Vec<DbColumn>, rusqlite::Error> {
+ ) -> Result<Vec<DbColumn>, SqliteError> {
  let mut column_names = conn.prepare(&format!(
  "SELECT * FROM pragma_table_info('{}');",
  table.name
@@ -160,7 +227,7 @@ impl SQLiteDatabase {
  Ok(columns)
  }
 
- fn get_constraint_info(&self, row: &Row) -> Result<DbConstraint, rusqlite::Error> {
+ fn get_constraint_info(&self, row: &Row) -> Result<DbConstraint, SqliteError> {
  let dbc = DbConstraint {
  name: row.get("index_name")?,
  column_name: row.get("column_name")?,
@@ -173,7 +240,7 @@ impl SQLiteDatabase {
  &self,
  conn: &Connection,
  table: &DbTable,
- ) -> Result<Vec<DbConstraint>, rusqlite::Error> {
+ ) -> Result<Vec<DbConstraint>, SqliteError> {
  let mut column_names = conn.prepare(&format!(
  "
  SELECT
@@ -202,7 +269,7 @@ impl SQLiteDatabase {
  Ok(constraints)
  }
 
- fn get_foreign_keys_info(&self, row: &Row) -> Result<DbForeignKey, rusqlite::Error> {
+ fn get_foreign_keys_info(&self, row: &Row) -> Result<DbForeignKey, SqliteError> {
  let dbc = DbForeignKey {
  column_name: row.get("from")?,
  ref_table: row.get("table")?,
@@ -215,7 +282,7 @@ impl SQLiteDatabase {
  &self,
  conn: &Connection,
  table: &DbTable,
- ) -> Result<Vec<DbForeignKey>, rusqlite::Error> {
+ ) -> Result<Vec<DbForeignKey>, SqliteError> {
  let mut column_names = conn.prepare(&format!(
  "SELECT p.`from`, p.`to`, p.`table` FROM pragma_foreign_key_list('{}') p",
  &table.name
@@ -231,7 +298,7 @@ impl SQLiteDatabase {
  Ok(foreign_keys)
  }
 
- fn get_index_info(&self, row: &Row) -> Result<DbIndex, rusqlite::Error> {
+ fn get_index_info(&self, row: &Row) -> Result<DbIndex, SqliteError> {
  let dbc = DbIndex {
  name: row.get("index_name")?,
  column_name: row.get("name")?,
@@ -244,7 +311,7 @@ impl SQLiteDatabase {
  &self,
  conn: &Connection,
  table: &DbTable,
- ) -> Result<Vec<DbIndex>, rusqlite::Error> {
+ ) -> Result<Vec<DbIndex>, SqliteError> {
  let mut column_names = conn.prepare(&format!(
  "
  SELECT
@@ -330,25 +397,28 @@ impl CustomValue for SQLiteDatabase {
  }
 }
 
-pub fn open_sqlite_db(path: &Path, call_span: Span) -> Result<Connection, nu_protocol::ShellError> {
- let path = path.to_string_lossy().to_string();
-
- Connection::open(path).map_err(|e| {
- ShellError::GenericError(
- "Failed to open SQLite database".into(),
- e.to_string(),
- Some(call_span),
- None,
- Vec::new(),
- )
- })
+pub fn open_sqlite_db(path: &Path, call_span: Span) -> Result<Connection, ShellError> {
+ if path.to_string_lossy() == MEMORY_DB {
+ open_connection_in_memory_custom()
+ } else {
+ let path = path.to_string_lossy().to_string();
+ Connection::open(path).map_err(|e| {
+ ShellError::GenericError(
+ "Failed to open SQLite database".into(),
+ e.to_string(),
+ Some(call_span),
+ None,
+ Vec::new(),
+ )
+ })
+ }
 }
 
 fn run_sql_query(
  conn: Connection,
  sql: &Spanned<String>,
  ctrlc: Option<Arc<AtomicBool>>,
-) -> Result<Value, rusqlite::Error> {
+) -> Result<Value, SqliteError> {
  let stmt = conn.prepare(&sql.item)?;
  prepared_statement_to_nu_list(stmt, sql.span, ctrlc)
 }
@@ -358,16 +428,16 @@ fn read_single_table(
  table_name: String,
  call_span: Span,
  ctrlc: Option<Arc<AtomicBool>>,
-) -> Result<Value, rusqlite::Error> {
+) -> Result<Value, SqliteError> {
  let stmt = conn.prepare(&format!("SELECT * FROM [{table_name}]"))?;
  prepared_statement_to_nu_list(stmt, call_span, ctrlc)
 }
 
 fn prepared_statement_to_nu_list(
- mut stmt: rusqlite::Statement,
+ mut stmt: Statement,
  call_span: Span,
  ctrlc: Option<Arc<AtomicBool>>,
-) -> Result<Value, rusqlite::Error> {
+) -> Result<Value, SqliteError> {
  let column_names = stmt
  .column_names()
  .iter()
@@ -403,7 +473,7 @@ fn read_entire_sqlite_db(
  conn: Connection,
  call_span: Span,
  ctrlc: Option<Arc<AtomicBool>>,
-) -> Result<Value, rusqlite::Error> {
+) -> Result<Value, SqliteError> {
  let mut tables = Record::new();
 
  let mut get_table_names =
@@ -455,9 +525,8 @@ pub fn convert_sqlite_value_to_nu_value(value: ValueRef, span: Span) -> Value {
 
 #[cfg(test)]
 mod test {
- use nu_protocol::record;
-
  use super::*;
+ use nu_protocol::record;
 
  #[test]
  fn can_read_empty_db() {
@@ -532,10 +601,23 @@ mod test {
  }
 }
 
+pub fn open_connection_in_memory_custom() -> Result<Connection, ShellError> {
+ let flags = OpenFlags::default();
+ Connection::open_with_flags(MEMORY_DB, flags).map_err(|err| {
+ ShellError::GenericError(
+ "Failed to open SQLite custom connection in memory".into(),
+ err.to_string(),
+ Some(Span::test_data()),
+ None,
+ Vec::new(),
+ )
+ })
+}
+
 pub fn open_connection_in_memory() -> Result<Connection, ShellError> {
  Connection::open_in_memory().map_err(|err| {
  ShellError::GenericError(
- "Failed to open SQLite connection in memory".into(),
+ "Failed to open SQLite standard connection in memory".into(),
  err.to_string(),
  Some(Span::test_data()),
  None,

crates/nu-command/src/default_context.rs

@@ -404,6 +404,20 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
  DateFormat,
  };
 
+ // Stor
+ #[cfg(feature = "sqlite")]
+ bind_command! {
+ Stor,
+ StorCreate,
+ StorDelete,
+ StorExport,
+ StorImport,
+ StorInsert,
+ StorOpen,
+ StorReset,
+ StorUpdate,
+ };
+
  working_set.render()
  };
 

crates/nu-command/src/lib.rs

@@ -23,6 +23,7 @@ mod random;
 mod removed;
 mod shells;
 mod sort_utils;
+mod stor;
 mod strings;
 mod system;
 mod viewers;
@@ -52,6 +53,8 @@ pub use random::*;
 pub use removed::*;
 pub use shells::*;
 pub use sort_utils::*;
+#[cfg(feature = "sqlite")]
+pub use stor::*;
 pub use strings::*;
 pub use system::*;
 pub use viewers::*;