forked from jqnatividad/qsv
-
Notifications
You must be signed in to change notification settings - Fork 0
/
diff.rs
212 lines (180 loc) · 7.39 KB
/
diff.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
static USAGE: &str = r#"
Creates the difference between two CSVs.
Note that diff does not support stdin. A file path is required for both arguments.
Usage:
qsv diff [options] [<input-left>] [<input-right>]
qsv diff --help
diff options:
--no-headers-left When set, the first row will be considered as part of
the left CSV to diff. (When not set, the
first row is the header row and will be skipped during
the diff. It will always appear in the output.)
--no-headers-right When set, the first row will be considered as part of
the right CSV to diff. (When not set, the
first row is the header row and will be skipped during
the diff. It will always appear in the output.)
--delimiter-left <arg> The field delimiter for reading CSV data on the left.
Must be a single character. (default: ,)
--delimiter-right <arg> The field delimiter for reading CSV data on the right.
Must be a single character. (default: ,)
--primary-key-idx <arg...> The column indices that uniquely identify a record
as a comma separated list of indices, e.g. 0,1,2.
(default: 0)
-j, --jobs <arg> The number of jobs to run in parallel.
When not set, the number of jobs is set to the number
of CPUs detected.
Common options:
-h, --help Display this message
-o, --output <file> Write output to <file> instead of stdout.
"#;
use std::io::{self, Read, Write};
use csv_diff::{csv_diff::CsvByteDiffBuilder, diff_row::DiffByteRecord};
use serde::Deserialize;
use crate::{
clitypes::CliError,
config::{Config, Delimiter},
util, CliResult,
};
#[derive(Deserialize)]
struct Args {
arg_input_left: Option<String>,
arg_input_right: Option<String>,
flag_output: Option<String>,
flag_jobs: Option<usize>,
flag_no_headers_left: bool,
flag_no_headers_right: bool,
flag_delimiter_left: Option<Delimiter>,
flag_delimiter_right: Option<Delimiter>,
flag_primary_key_idx: Option<String>,
}
pub fn run(argv: &[&str]) -> CliResult<()> {
let args: Args = util::get_args(USAGE, argv)?;
let rconfig_left = Config::new(&args.arg_input_left)
.delimiter(args.flag_delimiter_left)
.checkutf8(false)
.no_headers(args.flag_no_headers_left);
let rconfig_right = Config::new(&args.arg_input_right)
.delimiter(args.flag_delimiter_right)
.checkutf8(false)
.no_headers(args.flag_no_headers_right);
if rconfig_left.is_stdin() || rconfig_right.is_stdin() {
return fail_clierror!(
"diff does not support stdin. A file path is required for both arguments."
);
}
let primary_key_cols = match args.flag_primary_key_idx {
None => vec![0],
Some(s) => s
.split(',')
.map(str::parse::<usize>)
.collect::<Result<Vec<_>, _>>()
.map_err(|err| CliError::Other(err.to_string()))?,
};
let wtr = Config::new(&args.flag_output).writer()?;
let mut csv_rdr_left = rconfig_left.reader()?;
let mut csv_rdr_right = rconfig_right.reader()?;
// set RAYON_NUM_THREADS
util::njobs(args.flag_jobs);
let mut csv_diff_writer = CsvDiffWriter::new(wtr);
csv_diff_writer.write_headers(&mut csv_rdr_left, &mut csv_rdr_right)?;
let Ok(csv_diff) = CsvByteDiffBuilder::new()
.primary_key_columns(primary_key_cols)
.build() else {
return fail_clierror!("Cannot instantiate diff")
};
let diff_byte_records_iter = csv_diff.diff(csv_rdr_left.into(), csv_rdr_right.into());
Ok(csv_diff_writer.write_diff_byte_records(diff_byte_records_iter)?)
}
struct CsvDiffWriter<W: Write> {
csv_writer: csv::Writer<W>,
}
impl<W: Write> CsvDiffWriter<W> {
fn new(csv_writer: csv::Writer<W>) -> Self {
Self { csv_writer }
}
fn write_headers<R: Read>(
&mut self,
rdr_left: &mut csv::Reader<R>,
rdr_right: &mut csv::Reader<R>,
) -> csv::Result<()> {
match (rdr_left.has_headers(), rdr_right.has_headers()) {
(true, true) => {
let rdr_bh = rdr_left.byte_headers()?;
rdr_bh.write_diffresult_header(&mut self.csv_writer)?;
// we also read the headers from the right CSV, so that both readers end up
// before the actual records. Otherwise, it would lead to errors when we
// diff the CSVs, because the header of one CSV would have been read and the other
// not.
let _ = rdr_right.byte_headers()?;
}
(true, false) => {
let rdr_bh = rdr_left.byte_headers()?;
rdr_bh.write_diffresult_header(&mut self.csv_writer)?;
}
(false, true) => {
let rdr_bh = rdr_right.byte_headers()?;
rdr_bh.write_diffresult_header(&mut self.csv_writer)?;
}
// nothing to do, because there are no headers
(false, false) => {}
}
Ok(())
}
fn write_diff_byte_records(
&mut self,
diff_byte_records: impl IntoIterator<Item = csv::Result<DiffByteRecord>>,
) -> io::Result<()> {
for dbr in diff_byte_records {
self.write_diff_byte_record(&dbr?)?;
}
self.csv_writer.flush()?;
Ok(())
}
fn write_diff_byte_record(&mut self, diff_byte_record: &DiffByteRecord) -> csv::Result<()> {
let add_sign: &[u8] = &b"+"[..];
let remove_sign: &[u8] = &b"-"[..];
match diff_byte_record {
DiffByteRecord::Add(add) => {
let mut vec = vec![add_sign];
vec.extend(add.byte_record());
self.csv_writer.write_record(vec)
}
DiffByteRecord::Modify {
delete,
add,
// TODO: this should be used in the future to highlight the column where differences
// occur
field_indices: _field_indices,
} => {
let mut vec_del = vec![remove_sign];
vec_del.extend(delete.byte_record());
self.csv_writer.write_record(vec_del)?;
let mut vec_add = vec![add_sign];
vec_add.extend(add.byte_record());
self.csv_writer.write_record(vec_add)
}
DiffByteRecord::Delete(del) => {
let mut vec = vec![remove_sign];
vec.extend(del.byte_record());
self.csv_writer.write_record(vec)
}
}
}
}
trait WriteDiffResultHeader {
fn write_diffresult_header<W: Write>(&self, csv_writer: &mut csv::Writer<W>)
-> csv::Result<()>;
}
impl WriteDiffResultHeader for csv::ByteRecord {
fn write_diffresult_header<W: Write>(
&self,
csv_writer: &mut csv::Writer<W>,
) -> csv::Result<()> {
if !self.is_empty() {
let mut new_header = vec![&b"diffresult"[..]];
new_header.extend(self);
csv_writer.write_record(new_header)?;
}
Ok(())
}
}