Skip to content

Commit

Permalink
stub out casting datatypes in conversion records
Browse files Browse the repository at this point in the history
  • Loading branch information
hugopendlebury committed Feb 1, 2024
1 parent 85f5e72 commit 444ad6d
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 1 deletion.
2 changes: 1 addition & 1 deletion pythonApi/grib_to_arrow.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include "../src/exceptions/invalidcsvexception.hpp"
#include "../src/exceptions/invalidschemaexception.hpp"

#define USE_CMAKE
//#define USE_CMAKE

#ifdef USE_CMAKE
#include "../pybind11/include/pybind11/pybind11.h"
Expand Down
28 changes: 28 additions & 0 deletions src/gribreader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,34 @@ void GribReader::validateConversionFields(std::shared_ptr<arrow::Table> location
}
}

void GribReader::castConversionFields(std::shared_ptr<arrow::Table> locations, std::string table_name) {
auto table = locations.get();

std::vector<std::string> f64_cols = {"addition_value",
"subtraction_value",
"multiplication_value",
"division_value",
"ceiling_value"};

for (auto colName: f64_cols) {
auto col = table->GetColumnByName(colName).get();

cp::CastOptions castOptions;
castOptions.to_type = arrow::float64().get();

auto x = arrow::float64().get();
for (auto chunk : col->chunks()) {
auto arr = chunk.get();
auto result = cp::Cast(*arr, castOptions.to_type, castOptions);
if (result.ok()) {
auto converted = result.ValueOrDie();
//Ok it's converted do we need to update the table ?
}
}
}

}

GribReader GribReader::withLocations(std::string path){

//Reads a CSV with the location data and enriches it with a row_number / surrogate_key
Expand Down
1 change: 1 addition & 0 deletions src/gribreader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class GribReader
std::shared_ptr<arrow::Table> getTableFromCsv(std::string path, arrow::csv::ConvertOptions convertOptions);
arrow::Result<std::shared_ptr<arrow::Array>> createSurrogateKeyCol(long numberOfRows);
void validateConversionFields(std::shared_ptr<arrow::Table> locations, std::string table_name);
void castConversionFields(std::shared_ptr<arrow::Table> locations, std::string table_name) ;


};

0 comments on commit 444ad6d

Please sign in to comment.