Support parse_url function (apache#136)

* add parse_url

* fix code err

* fix code

* add suite tests

* add missing code

* refresh code

* add comment

Author: jackylee-ch

cpp/src/arrow/util/uri.cc

@@ -141,8 +141,12 @@ bool Uri::has_host() const { return IsTextRangeSet(impl_->uri_.hostText); }
 
 std::string Uri::port_text() const { return TextRangeToString(impl_->uri_.portText); }
 
+bool Uri::has_port() const { return IsTextRangeSet(impl_->uri_.portText); }
+
 int32_t Uri::port() const { return impl_->port_; }
 
+std::string Uri::user_info() const { return TextRangeToString(impl_->uri_.userInfo); }
+
 std::string Uri::username() const {
   auto userpass = TextRangeToView(impl_->uri_.userInfo);
   auto sep_pos = userpass.find_first_of(':');
@@ -194,6 +198,8 @@ std::string Uri::path() const {
 
 std::string Uri::query_string() const { return TextRangeToString(impl_->uri_.query); }
 
+bool Uri::has_query() const { return IsTextRangeSet(impl_->uri_.query); }
+
 Result<std::vector<std::pair<std::string, std::string>>> Uri::query_items() const {
   const auto& query = impl_->uri_.query;
   UriQueryListA* query_list;

cpp/src/arrow/util/uri.h

@@ -56,10 +56,14 @@ class ARROW_EXPORT Uri {
   /// The URI port number, as a string such as "80", or the empty string is the URI
   /// does not have a port number component.
   std::string port_text() const;
+  /// Whether the URI has an explicit port.
+  bool has_port() const;
   /// The URI port parsed as an integer, or -1 if the URI does not have a port
   /// number component.
   int32_t port() const;
 
+  /// The userInfo specified in the URI.
+  std::string user_info() const;
   /// The username specified in the URI.
   std::string username() const;
   /// The password specified in the URI.
@@ -70,6 +74,8 @@ class ARROW_EXPORT Uri {
 
   /// The URI query string
   std::string query_string() const;
+  /// Whether the URI has an explicit query.
+  bool has_query() const;
 
   /// The URI query items
   ///

cpp/src/gandiva/CMakeLists.txt

@@ -92,6 +92,7 @@ set(SRC_FILES
     regex_util.cc
     replace_holder.cc
     extract_holder.cc
+    parse_url_holder.cc
     selection_vector.cc
     tree_expr_builder.cc
     to_date_holder.cc
@@ -241,6 +242,7 @@ add_gandiva_test(internals-test
                  like_holder_test.cc
                  replace_holder_test.cc
                  extract_holder_test.cc
+                 parse_url_holder_test.cc
                  decimal_type_util_test.cc
                  random_generator_holder_test.cc
                  substr_index_holder_test.cc

cpp/src/gandiva/function_holder_registry.h

@@ -32,6 +32,7 @@
 #include "gandiva/replace_holder.h"
 #include "gandiva/rlike_holder.h"
 #include "gandiva/extract_holder.h"
+#include "gandiva/parse_url_holder.h"
 #include "gandiva/to_date_holder.h"
 #include "gandiva/translate_holder.h"
 #include "gandiva/substr_index_holder.h"
@@ -76,6 +77,7 @@ class FunctionHolderRegistry {
         {"rand", LAMBDA_MAKER(RandomGeneratorHolder)},
         {"regexp_replace", LAMBDA_MAKER(ReplaceHolder)},
         {"regexp_extract", LAMBDA_MAKER(ExtractHolder)},
+        {"parse_url", LAMBDA_MAKER(ParseUrlHolder)},
         {"translate", LAMBDA_MAKER(TranslateHolder)},
         {"substr_index", LAMBDA_MAKER(SubstrIndexHolder)}
     };

cpp/src/gandiva/function_registry_string.cc

@@ -221,6 +221,18 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
                      NativeFunction::kNeedsFunctionHolder |
                      NativeFunction::kCanReturnErrors),
 
+      NativeFunction("parse_url", {}, DataTypeVector{utf8(), utf8()}, utf8(),
+                     kResultNullIfNull, "gdv_fn_parse_url_utf8_utf8",
+                     NativeFunction::kNeedsContext |
+                     NativeFunction::kNeedsFunctionHolder |
+                     NativeFunction::kCanReturnErrors),
+
+      NativeFunction("parse_url", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(),
+                     kResultNullIfNull, "gdv_fn_parse_url_utf8_utf8_utf8",
+                     NativeFunction::kNeedsContext |
+                     NativeFunction::kNeedsFunctionHolder |
+                     NativeFunction::kCanReturnErrors),
+
       NativeFunction("lpad", {}, DataTypeVector{utf8(), int32(), utf8()}, utf8(),
                      kResultNullIfNull, "lpad_utf8_int32_utf8",
                      NativeFunction::kNeedsContext),

cpp/src/gandiva/gdv_function_stubs.cc

@@ -34,6 +34,7 @@
 #include "gandiva/replace_holder.h"
 #include "gandiva/rlike_holder.h"
 #include "gandiva/extract_holder.h"
+#include "gandiva/parse_url_holder.h"
 #include "gandiva/to_date_holder.h"
 #include "gandiva/translate_holder.h"
 #include "gandiva/substr_index_holder.h"
@@ -125,6 +126,48 @@ const char* gdv_fn_regexp_extract_utf8_utf8_int32(
   return (*holder)(context, data, data_len, idx, out_length);
 }
 
+const char* gdv_fn_parse_url_utf8_utf8(
+    int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len, bool in1_valid,
+    const char* part, int32_t part_len, bool in2_valid, bool* out_valid, int32_t* out_length) {
+  if (!in1_valid || !in2_valid) {
+    *out_valid = false;
+    *out_length = 0;
+    return reinterpret_cast<const char*>("");
+  }
+  gandiva::ExecutionContext* context = reinterpret_cast<gandiva::ExecutionContext*>(ptr);
+  gandiva::ParseUrlHolder* holder = reinterpret_cast<gandiva::ParseUrlHolder*>(holder_ptr);
+  auto res = (*holder)(context, data, data_len, part, part_len, out_length);
+  if (res == nullptr) {
+    *out_valid = false;
+    *out_length = 0;
+    return reinterpret_cast<const char*>("");
+  }
+  *out_valid = true;
+  return res;
+}
+
+const char* gdv_fn_parse_url_utf8_utf8_utf8(
+    int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len, bool in1_valid,
+    const char* part, int32_t part_len, bool in2_valid,
+    const char* pattern, int32_t pattern_len, bool in3_valid,
+    bool* out_valid, int32_t* out_length) {
+  if (!in1_valid || !in2_valid || !in3_valid) {
+    *out_valid = false;
+    *out_length = 0;
+    return reinterpret_cast<const char*>("");
+  }
+  gandiva::ExecutionContext* context = reinterpret_cast<gandiva::ExecutionContext*>(ptr);
+  gandiva::ParseUrlHolder* holder = reinterpret_cast<gandiva::ParseUrlHolder*>(holder_ptr);
+  auto res = (*holder)(context, data, data_len,  part, part_len, pattern, pattern_len, out_length);
+  if (res == nullptr) {
+    *out_valid = false;
+    *out_length = 0;
+    return reinterpret_cast<const char*>("");
+  }
+  *out_valid = true;
+  return res;
+}
+
 double gdv_fn_random(int64_t ptr) {
   gandiva::RandomGeneratorHolder* holder =
       reinterpret_cast<gandiva::RandomGeneratorHolder*>(ptr);
@@ -749,6 +792,41 @@ void ExportedStubFunctions::AddMappings(Engine* engine) const {
       "gdv_fn_regexp_extract_utf8_utf8_int32", types->i8_ptr_type() /*return_type*/, args,
       reinterpret_cast<void*>(gdv_fn_regexp_extract_utf8_utf8_int32));
 
+  // gdv_fn_parse_url_utf8_utf8
+  args = {types->i64_type(),       // int64_t ptr
+          types->i64_type(),       // int64_t holder_ptr
+          types->i8_ptr_type(),    // const char* data
+          types->i32_type(),       // int data_len
+          types->i1_type(),        // bool in1_validity
+          types->i8_ptr_type(),    // const char* part
+          types->i32_type(),       // int part_len
+          types->i1_type(),      // bool in2_validity
+          types->ptr_type(types->i8_type()),  // bool* out_valid
+          types->i32_ptr_type()};  // int32_t* out_length
+
+  engine->AddGlobalMappingForFunc(
+      "gdv_fn_parse_url_utf8_utf8", types->i8_ptr_type() /*return_type*/, args,
+      reinterpret_cast<void*>(gdv_fn_parse_url_utf8_utf8));
+
+  // gdv_fn_parse_url_utf8_utf8_utf8
+  args = {types->i64_type(),       // int64_t ptr
+          types->i64_type(),       // int64_t holder_ptr
+          types->i8_ptr_type(),    // const char* data
+          types->i32_type(),       // int data_len
+          types->i1_type(),        // bool in1_validity
+          types->i8_ptr_type(),    // const char* part
+          types->i32_type(),       // int part_len
+          types->i1_type(),        // bool in2_validity
+          types->i8_ptr_type(),    // const char* pattern
+          types->i32_type(),       // int pattern_len
+          types->i1_type(),        // bool in3_validity
+          types->ptr_type(types->i8_type()),  // bool* out_valid
+          types->i32_ptr_type()};  // int32_t* out_length
+
+  engine->AddGlobalMappingForFunc(
+      "gdv_fn_parse_url_utf8_utf8_utf8", types->i8_ptr_type() /*return_type*/, args,
+      reinterpret_cast<void*>(gdv_fn_parse_url_utf8_utf8_utf8));
+
   // gdv_fn_to_date_utf8_utf8
   args = {types->i64_type(),                   // int64_t execution_context
           types->i64_type(),                   // int64_t holder_ptr

cpp/src/gandiva/gdv_function_stubs.h

@@ -69,8 +69,18 @@ const char* gdv_fn_regexp_extract_utf8_utf8_int32(
     int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len,
     const char* /*pattern*/, int32_t /*pattern_len*/, const int32_t idx, int32_t* out_length);
 
-const char* gdv_fn_substr_index_utf8_utf8_int32(int64_t ptr, int64_t holder_ptr, const char* input, int in_len,
-                                         const char* delim, int delim_len, int count, int32_t* out_len);
+const char* gdv_fn_parse_url_utf8_utf8(
+        int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len,
+        const char* part, int32_t part_len, int32_t* out_length);
+
+const char* gdv_fn_parse_url_utf8_utf8(
+    int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len, bool in1_valid,
+    const char* part, int32_t part_len, bool in2_valid, bool* out_valid, int32_t* out_length);
+
+const char* gdv_fn_parse_url_utf8_utf8_utf8(
+    int64_t ptr, int64_t holder_ptr, const char* data, int32_t data_len, bool in1_valid,
+    const char* part, int32_t part_len, bool in2_valid,
+    const char* pattern, int32_t pattern_len, bool in3_valid, bool* out_valid, int32_t* out_length);
 
 int64_t gdv_fn_to_date_utf8_utf8_int32(int64_t context, int64_t ptr, const char* data,
                                        int data_len, bool in1_validity,

cpp/src/gandiva/parse_url_holder.cc

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gandiva/parse_url_holder.h"
+#include "gandiva/node.h"
+
+namespace gandiva {
+
+  Status ParseUrlHolder::Make(const FunctionNode &node, std::shared_ptr <ParseUrlHolder> *holder) {
+    return Make(holder);
+  }
+
+  Status ParseUrlHolder::Make(std::shared_ptr<ParseUrlHolder>* holder) {
+    *holder = std::shared_ptr<ParseUrlHolder>(new ParseUrlHolder());
+    return Status::OK();
+  }
+}  // namespace gandiva

cpp/src/gandiva/parse_url_holder.h

@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "arrow/status.h"
+#include "arrow/util/uri.h"
+#include "gandiva/execution_context.h"
+#include "gandiva/function_holder.h"
+#include "gandiva/node.h"
+#include "gandiva/visibility.h"
+
+namespace gandiva {
+
+  /// Function Holder for 'parse_url'
+  class GANDIVA_EXPORT ParseUrlHolder: public FunctionHolder {
+    public:
+    ~ParseUrlHolder() override = default;
+
+    // Invoked by function_holder_registry.h
+    static Status Make(const FunctionNode& node, std::shared_ptr<ParseUrlHolder>* holder);
+
+    static Status Make(std::shared_ptr<ParseUrlHolder>* holder);
+
+    const char * operator()(
+        ExecutionContext *ctx, const char * url, int32_t url_len,
+        const char * part, int32_t part_len, int32_t *out_length) {
+      std::string url_string(url, url_len);
+      std::string part_string(part, part_len);
+      arrow::internal::Uri uri;
+      std::string out;
+
+      auto status = uri.Parse(url_string);
+      if (!status.ok()) {
+        return nullptr;
+      }
+
+      if (part_string == "HOST") {
+        out = uri.host();
+      } else if (part_string == "PATH") {
+        out = uri.path();
+      } else if (part_string == "QUERY") {
+        out = uri.query_string();
+      } else if (part_string == "PROTOCOL") {
+        out = uri.scheme();
+      } else if (part_string == "FILE") {
+        if (uri.has_query()) {
+          out = uri.path() + "?" + uri.query_string();
+        } else {
+          out = uri.path();
+        }
+      } else if (part_string == "AUTHORITY") {
+        if (uri.has_port()) {
+          out = uri.host() + ":" + uri.port_text();
+        } else {
+          out = uri.host();
+        }
+      } else if (part_string == "USERINFO") {
+        out = uri.user_info();
+      } else {
+        return nullptr;
+      }
+
+      *out_length = static_cast<int32_t>(out.length());
+      if (*out_length == 0) {
+        return nullptr;
+      }
+
+      char *result_buffer = reinterpret_cast<char *>(ctx->arena()->Allocate(*out_length));
+      if (result_buffer == NULLPTR) {
+        ctx->set_error_msg("Could not allocate memory for result! Wrong result may be returned!");
+        *out_length = 0;
+        return nullptr;
+      }
+      memcpy(result_buffer, out.data(), *out_length);
+
+      return result_buffer;
+    }
+
+    // We only support plain pattern string here.
+    const char * operator()(
+        ExecutionContext *ctx, const char * url, int32_t url_len,
+        const char * part, int32_t part_len,
+        const char * pattern, int32_t pattern_len, int32_t *out_length) {
+      std::string url_string(url, url_len);
+      std::string part_string(part, part_len);
+      std::string pattern_string(pattern, pattern_len);
+      arrow::internal::Uri uri;
+      std::string out;
+
+      auto status = uri.Parse(url_string);
+      if (!status.ok()) {
+        return nullptr;
+      }
+
+      if (part_string != "QUERY" || !uri.has_query()) {
+        return nullptr;
+      } else {
+        std::unordered_map<std::string, std::string> queries;
+        const auto items = std::move(uri.query_items()).ValueUnsafe();
+        for (const auto& query : items) {
+          queries.emplace(query.first, query.second);
+        }
+
+        auto out_query = queries.find(pattern_string);
+        if (out_query == queries.end()) {
+          return nullptr;
+        }
+        out = out_query->second;
+
+        *out_length = static_cast<int32_t>(out.length());
+        if (*out_length == 0) {
+          return nullptr;
+        }
+
+        char *result_buffer = reinterpret_cast<char *>(ctx->arena()->Allocate(*out_length));
+        if (result_buffer == NULLPTR) {
+          ctx->set_error_msg("Could not allocate memory for result! Wrong result may be returned!");
+          *out_length = 0;
+          return nullptr;
+        }
+        memcpy(result_buffer, out.data(), *out_length);
+
+        return result_buffer;
+      }
+    }
+  };  // namespace gandiva
+}