Skip to content

Commit 62974d9

Browse files
zhouyuanCrystrixjvictorhugueninrodrigojdebem
authored
backport gandiva regexp related functions (apache#48)
* ARROW-11960: [C++][Gandiva] Support escape in LIKE Add gdv_fn_like_utf8_utf8_int8 function in Gandiva to support escape char in LIKE. An escape char is stored in an int8 type which is compatible with char type in C++. Closes apache#9700 from Crystrix/arrow-11960 Authored-by: crystrix <chenxi.li@live.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com> * ARROW-12567: [C++][Gandiva] Implement ILIKE SQL function Closes apache#10179 from jvictorhuguenin/feature/implement-sql-ilike and squashes the following commits: f160880 <frank400> Optimize holder constructor call 97e6e2d <frank400> Remove unnecessary Make method c2363b1 <frank400> Disable TryOptimize for ilike a484149 <frank400> Fix checkstyle on cmake file c6a8372 <frank400> Delete unnecessary holder 4be6cc6 <frank400> Fix redefined function b78085a <frank400> Fix miss include 2efd43e <frank400> Implement ilike function Authored-by: frank400 <j.victorhuguenin2018@gmail.com> Signed-off-by: Praveen <praveen@dremio.com> * ARROW-12410: [C++][Gandiva] Implement regexp_replace function on Gandiva Closes apache#10059 from rodrigojdebem/feature/implement-regexp-replace and squashes the following commits: baf2778 <rodrigojdebem> Add implementation for REGEXP_REPLACE Authored-by: rodrigojdebem <rodrigodebem1@gmail.com> Signed-off-by: Praveen <praveen@dremio.com> Co-authored-by: crystrix <chenxi.li@live.com> Co-authored-by: frank400 <j.victorhuguenin2018@gmail.com> Co-authored-by: rodrigojdebem <rodrigodebem1@gmail.com>
1 parent a1575c8 commit 62974d9

14 files changed

Lines changed: 713 additions & 12 deletions

File tree

‎cpp/src/gandiva/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ set(SRC_FILES
8686
literal_holder.cc
8787
projector.cc
8888
regex_util.cc
89+
replace_holder.cc
8990
selection_vector.cc
9091
tree_expr_builder.cc
9192
to_date_holder.cc
@@ -229,6 +230,7 @@ add_gandiva_test(internals-test
229230
to_date_holder_test.cc
230231
simple_arena_test.cc
231232
like_holder_test.cc
233+
replace_holder_test.cc
232234
decimal_type_util_test.cc
233235
random_generator_holder_test.cc
234236
hash_utils_test.cc

‎cpp/src/gandiva/function_holder_registry.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "gandiva/like_holder.h"
2929
#include "gandiva/node.h"
3030
#include "gandiva/random_generator_holder.h"
31+
#include "gandiva/replace_holder.h"
3132
#include "gandiva/to_date_holder.h"
3233

3334
namespace gandiva {
@@ -62,9 +63,11 @@ class FunctionHolderRegistry {
6263
static map_type& makers() {
6364
static map_type maker_map = {
6465
{"like", LAMBDA_MAKER(LikeHolder)},
66+
{"ilike", LAMBDA_MAKER(LikeHolder)},
6567
{"to_date", LAMBDA_MAKER(ToDateHolder)},
6668
{"random", LAMBDA_MAKER(RandomGeneratorHolder)},
6769
{"rand", LAMBDA_MAKER(RandomGeneratorHolder)},
70+
{"regexp_replace", LAMBDA_MAKER(ReplaceHolder)},
6871
};
6972
return maker_map;
7073
}

‎cpp/src/gandiva/function_registry_string.cc‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,14 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
160160
kResultNullIfNull, "gdv_fn_like_utf8_utf8",
161161
NativeFunction::kNeedsFunctionHolder),
162162

163+
NativeFunction("like", {}, DataTypeVector{utf8(), utf8(), utf8()}, boolean(),
164+
kResultNullIfNull, "gdv_fn_like_utf8_utf8_utf8",
165+
NativeFunction::kNeedsFunctionHolder),
166+
167+
NativeFunction("ilike", {}, DataTypeVector{utf8(), utf8()}, boolean(),
168+
kResultNullIfNull, "gdv_fn_ilike_utf8_utf8",
169+
NativeFunction::kNeedsFunctionHolder),
170+
163171
NativeFunction("ltrim", {}, DataTypeVector{utf8(), utf8()}, utf8(),
164172
kResultNullIfNull, "ltrim_utf8_utf8", NativeFunction::kNeedsContext),
165173

@@ -178,6 +186,12 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
178186
utf8(), kResultNullIfNull, "substr_utf8_int64",
179187
NativeFunction::kNeedsContext),
180188

189+
NativeFunction("regexp_replace", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(),
190+
kResultNullIfNull, "gdv_fn_regexp_replace_utf8_utf8",
191+
NativeFunction::kNeedsContext |
192+
NativeFunction::kNeedsFunctionHolder |
193+
NativeFunction::kCanReturnErrors),
194+
181195
NativeFunction("concatOperator", {}, DataTypeVector{utf8(), utf8()}, utf8(),
182196
kResultNullIfNull, "concatOperator_utf8_utf8",
183197
NativeFunction::kNeedsContext),

‎cpp/src/gandiva/gdv_function_stubs.cc‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "gandiva/like_holder.h"
3131
#include "gandiva/precompiled/types.h"
3232
#include "gandiva/random_generator_holder.h"
33+
#include "gandiva/replace_holder.h"
3334
#include "gandiva/to_date_holder.h"
3435

3536
/// Stub functions that can be accessed from LLVM or the pre-compiled library.
@@ -42,6 +43,31 @@ bool gdv_fn_like_utf8_utf8(int64_t ptr, const char* data, int data_len,
4243
return (*holder)(std::string(data, data_len));
4344
}
4445

46+
bool gdv_fn_like_utf8_utf8_utf8(int64_t ptr, const char* data, int data_len,
47+
const char* pattern, int pattern_len,
48+
const char* escape_char, int escape_char_len) {
49+
gandiva::LikeHolder* holder = reinterpret_cast<gandiva::LikeHolder*>(ptr);
50+
return (*holder)(std::string(data, data_len));
51+
}
52+
53+
bool gdv_fn_ilike_utf8_utf8(int64_t ptr, const char* data, int data_len,
54+
const char* pattern, int pattern_len) {
55+
gandiva::LikeHolder* holder = reinterpret_cast<gandiva::LikeHolder*>(ptr);
56+
return (*holder)(std::string(data, data_len));
57+
}
58+
59+
const char* gdv_fn_regexp_replace_utf8_utf8(
60+
int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len,
61+
const char* /*pattern*/, int32_t /*pattern_len*/, const char* replace_string,
62+
int32_t replace_string_len, int32_t* out_length) {
63+
gandiva::ExecutionContext* context = reinterpret_cast<gandiva::ExecutionContext*>(ptr);
64+
65+
gandiva::ReplaceHolder* holder = reinterpret_cast<gandiva::ReplaceHolder*>(holder_ptr);
66+
67+
return (*holder)(context, data, data_len, replace_string, replace_string_len,
68+
out_length);
69+
}
70+
4571
double gdv_fn_random(int64_t ptr) {
4672
gandiva::RandomGeneratorHolder* holder =
4773
reinterpret_cast<gandiva::RandomGeneratorHolder*>(ptr);
@@ -468,6 +494,45 @@ void ExportedStubFunctions::AddMappings(Engine* engine) const {
468494
types->i1_type() /*return_type*/, args,
469495
reinterpret_cast<void*>(gdv_fn_like_utf8_utf8));
470496

497+
// gdv_fn_like_utf8_utf8_utf8
498+
args = {types->i64_type(), // int64_t ptr
499+
types->i8_ptr_type(), // const char* data
500+
types->i32_type(), // int data_len
501+
types->i8_ptr_type(), // const char* pattern
502+
types->i32_type(), // int pattern_len
503+
types->i8_ptr_type(), // const char* escape_char
504+
types->i32_type()}; // int escape_char_len
505+
506+
engine->AddGlobalMappingForFunc("gdv_fn_like_utf8_utf8_utf8",
507+
types->i1_type() /*return_type*/, args,
508+
reinterpret_cast<void*>(gdv_fn_like_utf8_utf8_utf8));
509+
510+
// gdv_fn_ilike_utf8_utf8
511+
args = {types->i64_type(), // int64_t ptr
512+
types->i8_ptr_type(), // const char* data
513+
types->i32_type(), // int data_len
514+
types->i8_ptr_type(), // const char* pattern
515+
types->i32_type()}; // int pattern_len
516+
517+
engine->AddGlobalMappingForFunc("gdv_fn_ilike_utf8_utf8",
518+
types->i1_type() /*return_type*/, args,
519+
reinterpret_cast<void*>(gdv_fn_ilike_utf8_utf8));
520+
521+
// gdv_fn_regexp_replace_utf8_utf8
522+
args = {types->i64_type(), // int64_t ptr
523+
types->i64_type(), // int64_t holder_ptr
524+
types->i8_ptr_type(), // const char* data
525+
types->i32_type(), // int data_len
526+
types->i8_ptr_type(), // const char* pattern
527+
types->i32_type(), // int pattern_len
528+
types->i8_ptr_type(), // const char* replace_string
529+
types->i32_type(), // int32_t replace_string_len
530+
types->i32_ptr_type()}; // int32_t* out_length
531+
532+
engine->AddGlobalMappingForFunc(
533+
"gdv_fn_regexp_replace_utf8_utf8", types->i8_ptr_type() /*return_type*/, args,
534+
reinterpret_cast<void*>(gdv_fn_regexp_replace_utf8_utf8));
535+
471536
// gdv_fn_to_date_utf8_utf8
472537
args = {types->i64_type(), // int64_t execution_context
473538
types->i64_type(), // int64_t holder_ptr

‎cpp/src/gandiva/gdv_function_stubs.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,13 @@ using gdv_day_time_interval = int64_t;
4747
bool gdv_fn_like_utf8_utf8(int64_t ptr, const char* data, int data_len,
4848
const char* pattern, int pattern_len);
4949

50+
bool gdv_fn_like_utf8_utf8_utf8(int64_t ptr, const char* data, int data_len,
51+
const char* pattern, int pattern_len,
52+
const char* escape_char, int escape_char_len);
53+
54+
bool gdv_fn_ilike_utf8_utf8(int64_t ptr, const char* data, int data_len,
55+
const char* pattern, int pattern_len);
56+
5057
int64_t gdv_fn_to_date_utf8_utf8_int32(int64_t context, int64_t ptr, const char* data,
5158
int data_len, bool in1_validity,
5259
const char* pattern, int pattern_len,

‎cpp/src/gandiva/like_holder.cc‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ static bool IsArrowStringLiteral(arrow::Type::type type) {
6767
}
6868

6969
Status LikeHolder::Make(const FunctionNode& node, std::shared_ptr<LikeHolder>* holder) {
70-
ARROW_RETURN_IF(node.children().size() != 2,
71-
Status::Invalid("'like' function requires two parameters"));
70+
ARROW_RETURN_IF(node.children().size() != 2 && node.children().size() != 3,
71+
Status::Invalid("'like' function requires two or three parameters"));
7272

7373
auto literal = dynamic_cast<LiteralNode*>(node.children().at(1).get());
7474
ARROW_RETURN_IF(
@@ -81,7 +81,28 @@ Status LikeHolder::Make(const FunctionNode& node, std::shared_ptr<LikeHolder>* h
8181
Status::Invalid(
8282
"'like' function requires a string literal as the second parameter"));
8383

84-
return Make(arrow::util::get<std::string>(literal->holder()), holder);
84+
RE2::Options regex_op;
85+
if (node.descriptor()->name() == "ilike") {
86+
regex_op.set_case_sensitive(false); // set case-insensitive for ilike function.
87+
88+
return Make(arrow::util::get<std::string>(literal->holder()), holder, regex_op);
89+
}
90+
if (node.children().size() == 2) {
91+
return Make(arrow::util::get<std::string>(literal->holder()), holder);
92+
} else {
93+
auto escape_char = dynamic_cast<LiteralNode*>(node.children().at(2).get());
94+
ARROW_RETURN_IF(
95+
escape_char == nullptr,
96+
Status::Invalid("'like' function requires a literal as the third parameter"));
97+
98+
auto escape_char_type = escape_char->return_type()->id();
99+
ARROW_RETURN_IF(
100+
!IsArrowStringLiteral(escape_char_type),
101+
Status::Invalid(
102+
"'like' function requires a string literal as the third parameter"));
103+
return Make(arrow::util::get<std::string>(literal->holder()),
104+
arrow::util::get<std::string>(escape_char->holder()), holder);
105+
}
85106
}
86107

87108
Status LikeHolder::Make(const std::string& sql_pattern,
@@ -97,4 +118,39 @@ Status LikeHolder::Make(const std::string& sql_pattern,
97118
return Status::OK();
98119
}
99120

121+
Status LikeHolder::Make(const std::string& sql_pattern, const std::string& escape_char,
122+
std::shared_ptr<LikeHolder>* holder) {
123+
ARROW_RETURN_IF(escape_char.length() > 1,
124+
Status::Invalid("The length of escape char ", escape_char,
125+
" in 'like' function is greater than 1"));
126+
std::string pcre_pattern;
127+
if (escape_char.length() == 1) {
128+
ARROW_RETURN_NOT_OK(
129+
RegexUtil::SqlLikePatternToPcre(sql_pattern, escape_char.at(0), pcre_pattern));
130+
} else {
131+
ARROW_RETURN_NOT_OK(RegexUtil::SqlLikePatternToPcre(sql_pattern, pcre_pattern));
132+
}
133+
134+
auto lholder = std::shared_ptr<LikeHolder>(new LikeHolder(pcre_pattern));
135+
ARROW_RETURN_IF(!lholder->regex_.ok(),
136+
Status::Invalid("Building RE2 pattern '", pcre_pattern, "' failed"));
137+
138+
*holder = lholder;
139+
return Status::OK();
140+
}
141+
142+
Status LikeHolder::Make(const std::string& sql_pattern,
143+
std::shared_ptr<LikeHolder>* holder, RE2::Options regex_op) {
144+
std::string pcre_pattern;
145+
ARROW_RETURN_NOT_OK(RegexUtil::SqlLikePatternToPcre(sql_pattern, pcre_pattern));
146+
147+
std::shared_ptr<LikeHolder> lholder;
148+
lholder = std::shared_ptr<LikeHolder>(new LikeHolder(pcre_pattern, regex_op));
149+
150+
ARROW_RETURN_IF(!lholder->regex_.ok(),
151+
Status::Invalid("Building RE2 pattern '", pcre_pattern, "' failed"));
152+
153+
*holder = lholder;
154+
return Status::OK();
155+
}
100156
} // namespace gandiva

‎cpp/src/gandiva/like_holder.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@ class GANDIVA_EXPORT LikeHolder : public FunctionHolder {
3939

4040
static Status Make(const std::string& sql_pattern, std::shared_ptr<LikeHolder>* holder);
4141

42+
static Status Make(const std::string& sql_pattern, const std::string& escape_char,
43+
std::shared_ptr<LikeHolder>* holder);
44+
45+
static Status Make(const std::string& sql_pattern, std::shared_ptr<LikeHolder>* holder,
46+
RE2::Options regex_op);
47+
4248
// Try and optimise a function node with a "like" pattern.
4349
static const FunctionNode TryOptimize(const FunctionNode& node);
4450

@@ -48,6 +54,9 @@ class GANDIVA_EXPORT LikeHolder : public FunctionHolder {
4854
private:
4955
explicit LikeHolder(const std::string& pattern) : pattern_(pattern), regex_(pattern) {}
5056

57+
LikeHolder(const std::string& pattern, RE2::Options regex_op)
58+
: pattern_(pattern), regex_(pattern, regex_op) {}
59+
5160
std::string pattern_; // posix pattern string, to help debugging
5261
RE2 regex_; // compiled regex for the pattern
5362

0 commit comments

Comments
 (0)