Skip to content

Commit 6704cdf

Browse files
anthonylouisbsbPindikura Ravindra
anthonylouisbsb
authored and
Pindikura Ravindra
committed
ARROW-12595: [C++][Gandiva] Implement TO_HEX([binary] field), HEX, UNHEX and FROM_HEX([string]field] functions
Functions that are responsible to convert binary to hexadecimal string representations. Closes #10195 from anthonylouisbsb/feature/add-to-hex Authored-by: Anthony Louis <anthony@simbioseventures.com> Signed-off-by: Pindikura Ravindra <ravindra@dremio.com>
1 parent 587ce1d commit 6704cdf

5 files changed

Lines changed: 463 additions & 0 deletions

File tree

‎cpp/src/gandiva/function_registry_string.cc‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,21 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
435435
kResultNullIfNull, "gdv_fn_castVARBINARY_float64_int64",
436436
NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors),
437437

438+
NativeFunction("to_hex", {"hex"}, DataTypeVector{binary()}, utf8(),
439+
kResultNullIfNull, "to_hex_binary", NativeFunction::kNeedsContext),
440+
441+
NativeFunction("to_hex", {"hex"}, DataTypeVector{utf8()}, utf8(), kResultNullIfNull,
442+
"to_hex_binary", NativeFunction::kNeedsContext),
443+
444+
NativeFunction("to_hex", {"hex"}, DataTypeVector{int64()}, utf8(),
445+
kResultNullIfNull, "to_hex_int64", NativeFunction::kNeedsContext),
446+
447+
NativeFunction("to_hex", {"hex"}, DataTypeVector{int32()}, utf8(),
448+
kResultNullIfNull, "to_hex_int32", NativeFunction::kNeedsContext),
449+
450+
NativeFunction("from_hex", {"unhex"}, DataTypeVector{utf8()}, binary(),
451+
kResultNullIfNull, "from_hex_utf8", NativeFunction::kNeedsContext),
452+
438453
NativeFunction("split_part", {}, DataTypeVector{utf8(), utf8(), int32()}, utf8(),
439454
kResultNullIfNull, "split_part",
440455
NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors)};

‎cpp/src/gandiva/precompiled/string_ops.cc‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@
1616
// under the License.
1717

1818
// String functions
19+
#include "arrow/util/logging.h"
1920
#include "arrow/util/value_parsing.h"
2021

2122
extern "C" {
2223

2324
#include <algorithm>
25+
#include <cinttypes>
2426
#include <climits>
2527
#include <cstdio>
2628
#include <cstdlib>
@@ -2465,4 +2467,118 @@ const char* elt_int32_utf8_utf8_utf8_utf8_utf8(
24652467
return nullptr;
24662468
}
24672469
}
2470+
2471+
// Gets a binary object and returns its hexadecimal representation. That representation
2472+
// maps each byte in the input to a 2-length string containing a hexadecimal number.
2473+
// - Examples:
2474+
// - foo -> 666F6F = 66[f] 6F[o] 6F[o]
2475+
// - bar -> 626172 = 62[b] 61[a] 72[r]
2476+
FORCE_INLINE
2477+
const char* to_hex_binary(int64_t context, const char* text, int32_t text_len,
2478+
int32_t* out_len) {
2479+
if (text_len == 0) {
2480+
*out_len = 0;
2481+
return "";
2482+
}
2483+
2484+
auto ret =
2485+
reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, text_len * 2 + 1));
2486+
2487+
if (ret == nullptr) {
2488+
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
2489+
*out_len = 0;
2490+
return "";
2491+
}
2492+
2493+
uint32_t ret_index = 0;
2494+
uint32_t max_len = static_cast<uint32_t>(text_len) * 2;
2495+
uint32_t max_char_to_write = 4;
2496+
2497+
for (gdv_int32 i = 0; i < text_len; i++) {
2498+
DCHECK(ret_index >= 0 && ret_index < max_len);
2499+
2500+
int32_t ch = static_cast<int32_t>(text[i]) & 0xFF;
2501+
2502+
ret_index += snprintf(ret + ret_index, max_char_to_write, "%02X", ch);
2503+
}
2504+
2505+
*out_len = static_cast<int32_t>(ret_index);
2506+
return ret;
2507+
}
2508+
2509+
FORCE_INLINE
2510+
const char* to_hex_int64(int64_t context, int64_t data, int32_t* out_len) {
2511+
const int64_t hex_long_max_size = 2 * sizeof(int64_t);
2512+
auto ret =
2513+
reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, hex_long_max_size));
2514+
2515+
if (ret == nullptr) {
2516+
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
2517+
*out_len = 0;
2518+
return "";
2519+
}
2520+
snprintf(ret, hex_long_max_size + 1, "%" PRIX64, data);
2521+
2522+
*out_len = static_cast<int32_t>(strlen(ret));
2523+
return ret;
2524+
}
2525+
2526+
FORCE_INLINE
2527+
const char* to_hex_int32(int64_t context, int32_t data, int32_t* out_len) {
2528+
const int32_t max_size = 2 * sizeof(int32_t);
2529+
auto ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, max_size));
2530+
2531+
if (ret == nullptr) {
2532+
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
2533+
*out_len = 0;
2534+
return "";
2535+
}
2536+
snprintf(ret, max_size + 1, "%" PRIX32, data);
2537+
2538+
*out_len = static_cast<int32_t>(strlen(ret));
2539+
return ret;
2540+
}
2541+
2542+
FORCE_INLINE
2543+
const char* from_hex_utf8(int64_t context, const char* text, int32_t text_len,
2544+
int32_t* out_len) {
2545+
if (text_len == 0) {
2546+
*out_len = 0;
2547+
return "";
2548+
}
2549+
2550+
// the input string should have a length multiple of two
2551+
if (text_len % 2 != 0) {
2552+
gdv_fn_context_set_error_msg(
2553+
context, "Error parsing hex string, length was not a multiple of two.");
2554+
*out_len = 0;
2555+
return "";
2556+
}
2557+
2558+
char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, text_len / 2));
2559+
2560+
if (ret == nullptr) {
2561+
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
2562+
*out_len = 0;
2563+
return "";
2564+
}
2565+
2566+
// converting hex encoded string to normal string
2567+
int32_t j = 0;
2568+
for (int32_t i = 0; i < text_len; i += 2) {
2569+
char b1 = text[i];
2570+
char b2 = text[i + 1];
2571+
if (isxdigit(b1) && isxdigit(b2)) {
2572+
// [a-fA-F0-9]
2573+
ret[j++] = to_binary_from_hex(b1) * 16 + to_binary_from_hex(b2);
2574+
} else {
2575+
gdv_fn_context_set_error_msg(
2576+
context, "Error parsing hex string, one or more bytes are not valid.");
2577+
*out_len = 0;
2578+
return "";
2579+
}
2580+
}
2581+
*out_len = j;
2582+
return ret;
2583+
}
24682584
} // extern "C"

‎cpp/src/gandiva/precompiled/string_ops_test.cc‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1872,4 +1872,230 @@ TEST(TestStringOps, TestEltFunction) {
18721872
EXPECT_EQ(out_vality, true);
18731873
}
18741874

1875+
TEST(TestStringOps, TestToHex) {
1876+
gandiva::ExecutionContext ctx;
1877+
uint64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
1878+
int32_t out_len = 0;
1879+
int32_t in_len = 0;
1880+
const char* out_str;
1881+
1882+
in_len = 10;
1883+
char in_str[] = {0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x69, 0x6E, 0x67};
1884+
out_str = to_hex_binary(ctx_ptr, in_str, in_len, &out_len);
1885+
std::string output = std::string(out_str, out_len);
1886+
EXPECT_EQ(out_len, 2 * in_len);
1887+
EXPECT_EQ(output, "54657374537472696E67");
1888+
1889+
in_len = 0;
1890+
out_str = to_hex_binary(ctx_ptr, "", in_len, &out_len);
1891+
output = std::string(out_str, out_len);
1892+
EXPECT_EQ(out_len, 0);
1893+
EXPECT_EQ(output, "");
1894+
1895+
in_len = 1;
1896+
char in_str_one_char[] = {0x54};
1897+
out_str = to_hex_binary(ctx_ptr, in_str_one_char, in_len, &out_len);
1898+
output = std::string(out_str, out_len);
1899+
EXPECT_EQ(out_len, 2 * in_len);
1900+
EXPECT_EQ(output, "54");
1901+
1902+
in_len = 16;
1903+
char in_str_spaces[] = {0x54, 0x65, 0x73, 0x74, 0x20, 0x77, 0x69, 0x74,
1904+
0x68, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73};
1905+
out_str = to_hex_binary(ctx_ptr, in_str_spaces, in_len, &out_len);
1906+
output = std::string(out_str, out_len);
1907+
EXPECT_EQ(output, "54657374207769746820737061636573");
1908+
1909+
in_len = 20;
1910+
char in_str_break_line[] = {0x54, 0x65, 0x78, 0x74, 0x20, 0x77, 0x69, 0x74, 0x68, 0x0A,
1911+
0x62, 0x72, 0x65, 0x61, 0x6B, 0x20, 0x6C, 0x69, 0x6E, 0x65};
1912+
out_str = to_hex_binary(ctx_ptr, in_str_break_line, in_len, &out_len);
1913+
output = std::string(out_str, out_len);
1914+
EXPECT_EQ(out_len, 2 * in_len);
1915+
EXPECT_EQ(output, "5465787420776974680A627265616B206C696E65");
1916+
1917+
in_len = 27;
1918+
char in_str_with_num[] = {0x54, 0x65, 0x73, 0x74, 0x20, 0x77, 0x69, 0x74, 0x68,
1919+
0x20, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x73, 0x20,
1920+
0x31, 0x20, 0x2B, 0x20, 0x31, 0x20, 0x3D, 0x20, 0x32};
1921+
out_str = to_hex_binary(ctx_ptr, in_str_with_num, in_len, &out_len);
1922+
output = std::string(out_str, out_len);
1923+
EXPECT_EQ(out_len, 2 * in_len);
1924+
EXPECT_EQ(output, "546573742077697468206E756D626572732031202B2031203D2032");
1925+
1926+
in_len = 22;
1927+
char in_str_with_tabs[] = {0x09, 0x0A, 0x09, 0x0A, 0x09, 0x0A, 0x09, 0x0A,
1928+
0x0A, 0x0A, 0x09, 0x20, 0x61, 0x20, 0x6C, 0x65,
1929+
0x74, 0x74, 0x40, 0x5D, 0x65, 0x72};
1930+
out_str = to_hex_binary(ctx_ptr, in_str_with_tabs, in_len, &out_len);
1931+
output = std::string(out_str, out_len);
1932+
EXPECT_EQ(out_len, 2 * in_len);
1933+
EXPECT_EQ(output, "090A090A090A090A0A0A092061206C657474405D6572");
1934+
1935+
in_len = 22;
1936+
const char* binary_string =
1937+
"\x09\x0A\x09\x0A\x09\x0A\x09\x0A\x0A\x0A\x09\x20\x61\x20\x6C\x65\x74\x74\x40\x5D"
1938+
"\x65\x72";
1939+
out_str = to_hex_binary(ctx_ptr, binary_string, in_len, &out_len);
1940+
output = std::string(out_str, out_len);
1941+
EXPECT_EQ(out_len, 2 * in_len);
1942+
EXPECT_EQ(output, "090A090A090A090A0A0A092061206C657474405D6572");
1943+
}
1944+
1945+
TEST(TestStringOps, TestToHexInt64) {
1946+
gandiva::ExecutionContext ctx;
1947+
uint64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
1948+
int32_t out_len = 0;
1949+
const char* out_str;
1950+
1951+
int64_t max_data = INT64_MAX;
1952+
out_str = to_hex_int64(ctx_ptr, max_data, &out_len);
1953+
std::string output = std::string(out_str, out_len);
1954+
EXPECT_FALSE(ctx.has_error());
1955+
EXPECT_EQ(out_len, 16);
1956+
EXPECT_EQ(output, "7FFFFFFFFFFFFFFF");
1957+
ctx.Reset();
1958+
1959+
int64_t min_data = INT64_MIN;
1960+
out_str = to_hex_int64(ctx_ptr, min_data, &out_len);
1961+
output = std::string(out_str, out_len);
1962+
EXPECT_FALSE(ctx.has_error());
1963+
EXPECT_EQ(out_len, 16);
1964+
EXPECT_EQ(output, "8000000000000000");
1965+
ctx.Reset();
1966+
1967+
int64_t zero_data = 0;
1968+
out_str = to_hex_int64(ctx_ptr, zero_data, &out_len);
1969+
output = std::string(out_str, out_len);
1970+
EXPECT_FALSE(ctx.has_error());
1971+
EXPECT_EQ(out_len, 1);
1972+
EXPECT_EQ(output, "0");
1973+
ctx.Reset();
1974+
1975+
int64_t minus_zero_data = -0;
1976+
out_str = to_hex_int64(ctx_ptr, minus_zero_data, &out_len);
1977+
output = std::string(out_str, out_len);
1978+
EXPECT_FALSE(ctx.has_error());
1979+
EXPECT_EQ(out_len, 1);
1980+
EXPECT_EQ(output, "0");
1981+
ctx.Reset();
1982+
1983+
int64_t minus_one_data = -1;
1984+
out_str = to_hex_int64(ctx_ptr, minus_one_data, &out_len);
1985+
output = std::string(out_str, out_len);
1986+
EXPECT_FALSE(ctx.has_error());
1987+
EXPECT_EQ(out_len, 16);
1988+
EXPECT_EQ(output, "FFFFFFFFFFFFFFFF");
1989+
ctx.Reset();
1990+
1991+
int64_t one_data = 1;
1992+
out_str = to_hex_int64(ctx_ptr, one_data, &out_len);
1993+
output = std::string(out_str, out_len);
1994+
EXPECT_FALSE(ctx.has_error());
1995+
EXPECT_EQ(out_len, 1);
1996+
EXPECT_EQ(output, "1");
1997+
ctx.Reset();
1998+
}
1999+
2000+
TEST(TestStringOps, TestToHexInt32) {
2001+
gandiva::ExecutionContext ctx;
2002+
uint64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
2003+
int32_t out_len = 0;
2004+
const char* out_str;
2005+
2006+
int32_t max_data = INT32_MAX;
2007+
out_str = to_hex_int32(ctx_ptr, max_data, &out_len);
2008+
std::string output = std::string(out_str, out_len);
2009+
EXPECT_FALSE(ctx.has_error());
2010+
EXPECT_EQ(out_len, 8);
2011+
EXPECT_EQ(output, "7FFFFFFF");
2012+
ctx.Reset();
2013+
2014+
int32_t min_data = INT32_MIN;
2015+
out_str = to_hex_int32(ctx_ptr, min_data, &out_len);
2016+
output = std::string(out_str, out_len);
2017+
EXPECT_FALSE(ctx.has_error());
2018+
EXPECT_EQ(out_len, 8);
2019+
EXPECT_EQ(output, "80000000");
2020+
ctx.Reset();
2021+
2022+
int32_t zero_data = 0;
2023+
out_str = to_hex_int32(ctx_ptr, zero_data, &out_len);
2024+
output = std::string(out_str, out_len);
2025+
EXPECT_FALSE(ctx.has_error());
2026+
EXPECT_EQ(out_len, 1);
2027+
EXPECT_EQ(output, "0");
2028+
ctx.Reset();
2029+
2030+
int32_t minus_zero_data = -0;
2031+
out_str = to_hex_int32(ctx_ptr, minus_zero_data, &out_len);
2032+
output = std::string(out_str, out_len);
2033+
EXPECT_FALSE(ctx.has_error());
2034+
EXPECT_EQ(out_len, 1);
2035+
EXPECT_EQ(output, "0");
2036+
ctx.Reset();
2037+
2038+
int32_t minus_one_data = -1;
2039+
out_str = to_hex_int32(ctx_ptr, minus_one_data, &out_len);
2040+
output = std::string(out_str, out_len);
2041+
EXPECT_FALSE(ctx.has_error());
2042+
EXPECT_EQ(out_len, 8);
2043+
EXPECT_EQ(output, "FFFFFFFF");
2044+
ctx.Reset();
2045+
2046+
int32_t one_data = 1;
2047+
out_str = to_hex_int32(ctx_ptr, one_data, &out_len);
2048+
output = std::string(out_str, out_len);
2049+
EXPECT_FALSE(ctx.has_error());
2050+
EXPECT_EQ(out_len, 1);
2051+
EXPECT_EQ(output, "1");
2052+
ctx.Reset();
2053+
}
2054+
2055+
TEST(TestStringOps, TestFromHex) {
2056+
gandiva::ExecutionContext ctx;
2057+
uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
2058+
gdv_int32 out_len = 0;
2059+
const char* out_str;
2060+
2061+
out_str = from_hex_utf8(ctx_ptr, "414243", 6, &out_len);
2062+
std::string output = std::string(out_str, out_len);
2063+
EXPECT_EQ(output, "ABC");
2064+
2065+
out_str = from_hex_utf8(ctx_ptr, "", 0, &out_len);
2066+
output = std::string(out_str, out_len);
2067+
EXPECT_EQ(output, "");
2068+
2069+
out_str = from_hex_utf8(ctx_ptr, "41", 2, &out_len);
2070+
output = std::string(out_str, out_len);
2071+
EXPECT_EQ(output, "A");
2072+
2073+
out_str = from_hex_utf8(ctx_ptr, "6d6D", 4, &out_len);
2074+
output = std::string(out_str, out_len);
2075+
EXPECT_EQ(output, "mm");
2076+
2077+
out_str = from_hex_utf8(ctx_ptr, "6f6d", 4, &out_len);
2078+
output = std::string(out_str, out_len);
2079+
EXPECT_EQ(output, "om");
2080+
2081+
out_str = from_hex_utf8(ctx_ptr, "4f4D", 4, &out_len);
2082+
output = std::string(out_str, out_len);
2083+
EXPECT_EQ(output, "OM");
2084+
2085+
out_str = from_hex_utf8(ctx_ptr, "T", 1, &out_len);
2086+
output = std::string(out_str, out_len);
2087+
EXPECT_EQ(output, "");
2088+
EXPECT_THAT(
2089+
ctx.get_error(),
2090+
::testing::HasSubstr("Error parsing hex string, length was not a multiple of"));
2091+
ctx.Reset();
2092+
2093+
out_str = from_hex_utf8(ctx_ptr, "\\x41\\x42\\x43", 12, &out_len);
2094+
output = std::string(out_str, out_len);
2095+
EXPECT_EQ(output, "");
2096+
EXPECT_THAT(
2097+
ctx.get_error(),
2098+
::testing::HasSubstr("Error parsing hex string, one or more bytes are not valid."));
2099+
ctx.Reset();
2100+
}
18752101
} // namespace gandiva

0 commit comments

Comments
 (0)