Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Internal pataching to 'st distance' #781

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,12 @@ if(ENABLE_CUDA)
else()
if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0")
message(STATUS "CMake 3.18+, Setting CUDA_ARCHITECTURES.")
set(CMAKE_CUDA_ARCHITECTURES
35-virtual
if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.0.0)
set(CMAKE_CUDA_ARCHITECTURES 35-virtual)
else()
set(CMAKE_CUDA_ARCHITECTURES "")
endif()
list(APPEND CMAKE_CUDA_ARCHITECTURES
50-virtual
60-virtual
70-virtual
Expand All @@ -258,8 +262,12 @@ if(ENABLE_CUDA)
message(STATUS "CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
else()
message(STATUS "CMake 3.17 or under, setting CUDA architecture flags manually.")
set(CUDA_COMPILATION_ARCH
-gencode=arch=compute_35,code=compute_35;
if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.0.0)
set(CUDA_COMPILATION_ARCH -gencode=arch=compute_35,code=compute_35;)
else()
set(CUDA_COMPILATION_ARCH "")
endif()
list(APPEND CUDA_COMPILATION_ARCH
-gencode=arch=compute_50,code=compute_50;
-gencode=arch=compute_60,code=compute_60;
-gencode=arch=compute_70,code=compute_70;
Expand Down
6 changes: 2 additions & 4 deletions DataMgr/ForeignStorage/CsvFileBufferParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ ParseBufferResult CsvFileBufferParser::parseBuffer(ParseBufferRequest& request,
std::vector<std::unique_ptr<char[]>>
tmp_buffers; // holds string w/ removed escape chars, etc
const char* line_start = p;
row_index_plus_one++;
bool incorrect_column_count = false;
p = import_export::delimited_parser::get_row(p,
thread_buf_end,
buf_end,
Expand All @@ -177,10 +179,6 @@ ParseBufferResult CsvFileBufferParser::parseBuffer(ParseBufferRequest& request,
tmp_buffers,
try_single_thread,
!columns_are_pre_filtered);

row_index_plus_one++;

bool incorrect_column_count = false;
try {
validate_expected_column_count(row, num_cols, point_cols, file_path);
} catch (const ForeignStorageException& e) {
Expand Down
46 changes: 32 additions & 14 deletions ImportExport/DelimitedParserUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,18 @@ inline void trim_space(const char*& field_begin, const char*& field_end) {
inline void trim_quotes(const char*& field_begin,
const char*& field_end,
const import_export::CopyParams& copy_params) {
if (copy_params.quoted && field_end - field_begin > 0 &&
*field_begin == copy_params.quote) {
++field_begin;
auto quote_begin = field_begin, quote_end = field_end;
if (copy_params.quoted) {
trim_space(quote_begin, quote_end);
}
if (copy_params.quoted && field_end - field_begin > 0 &&
*(field_end - 1) == copy_params.quote) {
--field_end;
if (copy_params.quoted && quote_end - quote_begin > 0) {
if (*quote_begin == copy_params.quote && *(quote_end - 1) == copy_params.quote) {
field_begin = ++quote_begin;
field_end = (quote_begin == quote_end) ? quote_end : --quote_end;
} else {
throw import_export::delimited_parser::DelimitedParserException(
"Unable to trim quotes.");
}
}
}
} // namespace
Expand Down Expand Up @@ -123,13 +128,26 @@ size_t find_end(const char* buffer,
if (last_line_delim_pos <= 0) {
size_t excerpt_length = std::min<size_t>(50, size);
std::string buffer_excerpt{buffer, buffer + excerpt_length};
std::string error_message =
"Unable to find an end of line character after reading " + std::to_string(size) +
" characters. Please ensure that the correct \"line_delimiter\" option is "
"specified or update the \"buffer_size\" option appropriately. Row number: " +
std::to_string(buffer_first_row_index + 1) +
". First few characters in row: " + buffer_excerpt;
throw InsufficientBufferSizeException{error_message};
if (in_quote) {
std::string quote(1, copy_params.quote);
std::string error_message =
"Unable to find a matching end quote for the quote character '" + quote +
"' after reading " + std::to_string(size) +
" characters. Please ensure that all data fields are correctly formatted "
"or update the \"buffer_size\" option appropriately. Row number: " +
std::to_string(buffer_first_row_index + 1) +
". First few characters in row: " + buffer_excerpt;
throw InsufficientBufferSizeException{error_message};
} else {
std::string error_message =
"Unable to find an end of line character after reading " +
std::to_string(size) +
" characters. Please ensure that the correct \"line_delimiter\" option is "
"specified or update the \"buffer_size\" option appropriately. Row number: " +
std::to_string(buffer_first_row_index + 1) +
". First few characters in row: " + buffer_excerpt;
throw InsufficientBufferSizeException{error_message};
}
}

return last_line_delim_pos + 1;
Expand Down Expand Up @@ -244,10 +262,10 @@ const char* get_row(const char* buf,
}
const char* field_begin = field_buf;
const char* field_end = field_buf + j;
trim_quotes(field_begin, field_end, copy_params);
if (copy_params.trim_spaces) {
trim_space(field_begin, field_end);
}
trim_quotes(field_begin, field_end, copy_params);
row.emplace_back(field_begin, field_end - field_begin);
}
field = p + 1;
Expand Down
5 changes: 5 additions & 0 deletions ImportExport/DelimitedParserUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ class InsufficientBufferSizeException : public std::runtime_error {
: std::runtime_error(message) {}
};

class DelimitedParserException : public std::runtime_error {
public:
DelimitedParserException(const std::string& message) : std::runtime_error(message) {}
};

/**
* @brief Finds the closest possible row beginning in the given buffer.
*
Expand Down
2 changes: 1 addition & 1 deletion ImportExport/Importer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2020,6 +2020,7 @@ static ImportStatus import_thread_delimited(
row.clear();
std::vector<std::unique_ptr<char[]>>
tmp_buffers; // holds string w/ removed escape chars, etc
row_index_plus_one++;
if (DEBUG_TIMING) {
us = measure<std::chrono::microseconds>::execution([&]() {
p = import_export::delimited_parser::get_row(p,
Expand All @@ -2044,7 +2045,6 @@ static ImportStatus import_thread_delimited(
try_single_thread,
true);
}
row_index_plus_one++;
// Each POINT could consume two separate coords instead of a single WKT
if (row.size() < num_cols || (num_cols + point_cols) < row.size()) {
thread_import_status.rows_rejected++;
Expand Down
Loading