Merge pull request #1867 from jqnatividad/1864-select-reverse-sort

`select`: add `--sort`, `--random` & `--seed` options; also add 9999 sentinel value to indicate last column

src/cmd/select.rs

@@ -21,6 +21,20 @@ selected using regular expressions.
  Select the third column named 'Foo':
  $ qsv select 'Foo[2]'
 
+ Select the first and last columns, 9999 is a special index for the last column:
+ $ qsv select 1,9999
+
+ Reverse the order of columns:
+ $ qsv select 9999-1
+
+ Sort the columns lexicographically. Note that you must provide a dummy selector:
+ $ qsv select 1 --sort
+
+ Randomly shuffle the columns:
+ $ qsv select 1 --random
+ # with a seed
+ $ qsv select 1 --random --seed 42
+
  Select columns using a regex using '/<regex>/':
  $ qsv select /^a/
  $ qsv select '/^.*\d.*$/'
@@ -39,6 +53,16 @@ Usage:
  qsv select [options] [--] <selection> [<input>]
  qsv select --help
 
+select options:
+These options only apply to the `select` command, not the `--select` flag in other commands.
+Be sure to provide a dummy selector (e.g. '1') to avoid command-line parsing errors.
+
+ -R, --random Randomly reorder the columns. 
+ --seed <number> Seed for the random number generator.
+
+ -S, --sort Sort the columns lexicographically, i.e. by their
+ byte values.
+
 Common options:
  -h, --help Display this message
  -o, --output <file> Write output to <file> instead of stdout.
@@ -49,6 +73,7 @@ Common options:
  Must be a single character. (default: ,)
 "#;
 
+use rand::{seq::SliceRandom, SeedableRng};
 use serde::Deserialize;
 
 use crate::{
@@ -61,13 +86,76 @@ use crate::{
 struct Args {
  arg_input: Option<String>,
  arg_selection: SelectColumns,
+ flag_random: bool,
+ flag_seed: Option<u64>,
+ flag_sort: bool,
  flag_output: Option<String>,
  flag_no_headers: bool,
  flag_delimiter: Option<Delimiter>,
 }
 
 pub fn run(argv: &[&str]) -> CliResult<()> {
- let args: Args = util::get_args(USAGE, argv)?;
+ let mut args: Args = util::get_args(USAGE, argv)?;
+
+ if args.flag_sort && args.flag_random {
+ return fail_clierror!("Cannot use both --random and --sort flags.");
+ }
+
+ if args.flag_random {
+ // get the number of columns
+ let num_cols = Config::new(&args.arg_input)
+ .delimiter(args.flag_delimiter)
+ .no_headers(true)
+ .reader()?
+ .byte_headers()?
+ .len();
+
+ // make a vector of the column indices (1-indexed).
+ let mut original_selection: Vec<usize> = (1..=num_cols).collect();
+
+ // Use seed if it is provided.
+ let mut rng = if let Some(seed) = args.flag_seed {
+ rand::rngs::StdRng::seed_from_u64(seed) // DevSkim: ignore DS148264
+ } else {
+ rand::rngs::StdRng::from_entropy()
+ };
+
+ // Shuffle the vector of column indices.
+ original_selection.shuffle(&mut rng);
+
+ // Convert the shuffled indices into a comma-separated string.
+ let randomized_selection = original_selection
+ .into_iter()
+ .map(|i| i.to_string())
+ .collect::<Vec<String>>()
+ .join(",");
+
+ // Parse the shuffled string into a SelectColumns object.
+ args.arg_selection = SelectColumns::parse(&randomized_selection)?;
+ }
+
+ if args.flag_sort {
+ // get the headers
+ let headers = Config::new(&args.arg_input)
+ .delimiter(args.flag_delimiter)
+ .reader()?
+ .byte_headers()?
+ .clone();
+
+ // sort the headers lexicographically
+ let mut sorted_headers = Vec::with_capacity(headers.len());
+ sorted_headers.extend(headers.iter().map(<[u8]>::to_vec));
+ sorted_headers.sort_unstable();
+
+ // make a comma-separated string of the sorted, quoted headers
+ let sorted_selection = sorted_headers
+ .iter()
+ .map(|h| format!("\"{}\"", String::from_utf8_lossy(h)))
+ .collect::<Vec<String>>()
+ .join(",");
+
+ args.arg_selection = SelectColumns::parse(&sorted_selection)?;
+ }
 
  let rconfig = Config::new(&args.arg_input)
  .delimiter(args.flag_delimiter)

src/select.rs

@@ -318,10 +318,14 @@ impl OneSelector {
  } else {
  first_record.len() - 1
  }),
- OneSelector::Index(i) => {
+ OneSelector::Index(mut i) => {
  if first_record.is_empty() {
  return fail!("Input is empty.");
  }
+ // 9999 is a sentinel value that means "last column".
+ if i == 9999 {
+ i = first_record.len();
+ }
  if i < 1 || i > first_record.len() {
  fail_format!(
  "Selector index {i} is out of bounds. Index must be >= 1 and <= {}.",

tests/test_select.rs

@@ -178,6 +178,14 @@ select_test!(
  ["a", "b", "d", "e"]
 );
 
+select_test!(
+ select_reverse_sentinel,
+ r#"9999-1"#,
+ "5-1",
+ ["h1", "h4", "h[]3", "h2", "h1"],
+ ["e", "d", "c", "b", "a"]
+);
+
 select_test_err!(select_err_unknown_header, "done");
 select_test_err!(select_err_oob_low, "0");
 select_test_err!(select_err_oob_high, "6");
@@ -193,3 +201,82 @@ select_test_err!(select_err_regex_nomatch, "/nomatch/");
 select_test_err!(select_err_regex_invalid, "/?/");
 select_test_err!(select_err_regex_empty, "//");
 select_test_err!(select_err_regex_triple_slash, "///");
+
+fn unsorted_data(headers: bool) -> Vec<Vec<String>> {
+ let mut rows = vec![
+ svec![
+ "value1", "value2", "value3", "value4", "value5", "value6", "value7", "value8",
+ "value9", "value10"
+ ],
+ svec!["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"],
+ svec![
+ "value10", "value9", "value8", "value7", "value6", "value5", "value4", "value3",
+ "value2", "value1"
+ ],
+ ];
+ if headers {
+ rows.insert(
+ 0,
+ svec![
+ "Günther", "Alice", "Çemil", "Đan", "Fátima", "Héctor", "İbrahim", "Bob", "Jürgen",
+ "Élise"
+ ],
+ );
+ }
+ rows
+}
+
+#[test]
+fn test_select_sort() {
+ let wrk = Workdir::new("test_select_sort");
+ wrk.create("data.csv", unsorted_data(true));
+ let mut cmd = wrk.command("select");
+ cmd.arg("1").arg("--sort").arg("data.csv");
+ let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+
+ let expected = vec![
+ svec![
+ "Alice", "Bob", "Fátima", "Günther", "Héctor", "Jürgen", "Çemil", "Élise", "Đan",
+ "İbrahim"
+ ],
+ svec![
+ "value2", "value8", "value5", "value1", "value6", "value9", "value3", "value10",
+ "value4", "value7"
+ ],
+ svec!["2", "8", "5", "1", "6", "9", "3", "10", "4", "7"],
+ svec![
+ "value9", "value3", "value6", "value10", "value5", "value2", "value8", "value1",
+ "value7", "value4"
+ ],
+ ];
+ assert_eq!(got, expected);
+}
+
+#[test]
+fn test_select_random_seeded() {
+ let wrk = Workdir::new("test_select_random_seeded");
+ wrk.create("data.csv", unsorted_data(true));
+ let mut cmd = wrk.command("select");
+ cmd.arg("1")
+ .arg("--random")
+ .args(["--seed", "42"])
+ .arg("data.csv");
+ let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+
+ let expected = vec![
+ svec![
+ "Bob", "Đan", "Élise", "Héctor", "Günther", "Jürgen", "İbrahim", "Fátima", "Çemil",
+ "Alice"
+ ],
+ svec![
+ "value8", "value4", "value10", "value6", "value1", "value9", "value7", "value5",
+ "value3", "value2"
+ ],
+ svec!["8", "4", "10", "6", "1", "9", "7", "5", "3", "2"],
+ svec![
+ "value3", "value7", "value1", "value5", "value10", "value2", "value4", "value6",
+ "value8", "value9"
+ ],
+ ];
+ assert_eq!(got, expected);
+}