[POAE7-2765] extended expression build and validate (#361)

* [POAE7-2765] extended expression build and validate

* [POAE7-2842] eu generation from extended expression

* rebase

* fix build

Author: yma11

cpp/CMakeLists.txt

@@ -142,14 +142,20 @@ if(BDTK_ENABLE_CIDER)
   #
   # Subdirectories
   #
+  # include_directories(src/cider) include_directories(src/cider/include)
+  # add_subdirectory(src/cider)
+
+  if(CIDER_ENABLE_VELOX)
+    add_subdirectory(thirdparty/velox)
+  endif()
+
   include_directories(src/cider)
   include_directories(src/cider/include)
   add_subdirectory(src/cider)
-
   if(CIDER_ENABLE_VELOX)
-    add_subdirectory(thirdparty/velox)
     add_subdirectory(src/cider-velox)
   endif()
+
 endif()
 
 if(BDTK_ENABLE_ICL)

cpp/src/cider/examples/CMakeLists.txt

@@ -18,8 +18,8 @@
 # under the License.
 
 add_executable(ExpressionEvalExample ExpressionEvalExample.cpp)
-target_link_libraries(ExpressionEvalExample test_utils cider_expr_builder cider
-                      substrait)
+target_link_libraries(ExpressionEvalExample dl cider_util test_utils
+                      cider_expr_builder substrait)
 
 add_executable(CiderHashTableExample CiderHashTableExample.cpp)
 target_link_libraries(CiderHashTableExample cider cider_hashtable_join)

cpp/src/cider/exec/plan/builder/SubstraitExprBuilder.cpp

@@ -37,6 +37,46 @@ SubstraitExprBuilder::SubstraitExprBuilder(std::vector<std::string> names,
   schema_ = makeNamedStruct(names, types);
 }
 
+::substrait::ExtendedExpression* SubstraitExprBuilder::build(
+    std::vector<ScalarExprRef> expr_refs) {
+  ::substrait::ExtendedExpression* ext_expr = new ::substrait::ExtendedExpression();
+  for (auto expr_ref : expr_refs) {
+    auto referred_expr = ext_expr->add_referred_expr();
+    referred_expr->set_allocated_expression(expr_ref.expr);
+    referred_expr->add_output_names(expr_ref.output_names);
+  }
+  using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI;
+  // Currently we don't introduce any substrait extension YAML files, so always
+  // only have one dummy URI.
+  SimpleExtensionURI* extensionUri = ext_expr->add_extension_uris();
+  extensionUri->set_extension_uri_anchor(1);
+  for (auto func_info : funcs_info_) {
+    ext_expr->add_extensions()->set_allocated_extension_function(func_info);
+  }
+  ext_expr->set_allocated_base_schema(schema_);
+  return ext_expr;
+}
+
+::substrait::ExtendedExpression* SubstraitExprBuilder::build(
+    std::vector<AggregationExprRef> expr_refs) {
+  ::substrait::ExtendedExpression* ext_expr = new ::substrait::ExtendedExpression();
+  for (auto expr_ref : expr_refs) {
+    auto referred_expr = ext_expr->add_referred_expr();
+    referred_expr->set_allocated_measure(expr_ref.expr);
+    referred_expr->add_output_names(expr_ref.output_names);
+  }
+  using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI;
+  // Currently we don't introduce any substrait extension YAML files, so always
+  // only have one dummy URI.
+  SimpleExtensionURI* extensionUri = ext_expr->add_extension_uris();
+  extensionUri->set_extension_uri_anchor(1);
+  for (auto func_info : funcs_info_) {
+    ext_expr->add_extensions()->set_allocated_extension_function(func_info);
+  }
+  ext_expr->set_allocated_base_schema(schema_);
+  return ext_expr;
+}
+
 ::substrait::NamedStruct* SubstraitExprBuilder::makeNamedStruct(
     std::vector<std::string> names,
     std::vector<::substrait::Type*> types) {
@@ -120,7 +160,7 @@ SubstraitExprBuilder::makeFunc(std::string func_name,
   func->set_name(func_sig);
   func->set_function_anchor(func_anchor_);
   func_anchor_ += 1;
-  func->set_extension_uri_reference(2);
+  func->set_extension_uri_reference(1);
   funcs_info_.push_back(func);
   return func;
 }

cpp/src/cider/exec/plan/builder/SubstraitExprBuilder.h

@@ -26,13 +26,19 @@
 
 #pragma once
 #include "substrait/algebra.pb.h"
+#include "substrait/extended_expression.pb.h"
 #include "substrait/type.pb.h"
 
-struct ExpressionReference {
+struct ScalarExprRef {
   ::substrait::Expression* expr;
   std::string output_names;
 };
 
+struct AggregationExprRef {
+  ::substrait::AggregateFunction* expr;
+  std::string output_names;
+};
+
 class SubstraitExprBuilder {
  public:
   SubstraitExprBuilder(std::vector<std::string> names,
@@ -42,8 +48,9 @@ class SubstraitExprBuilder {
 
   ::substrait::Expression* makeFieldReference(std::string name);
 
-  // TODO: (yma11) build extended expression
-  // ::substrait::ExtendedExpression* build(std::vector<ExpressionReference> expr_refs);
+  ::substrait::ExtendedExpression* build(std::vector<ScalarExprRef> expr_refs);
+
+  ::substrait::ExtendedExpression* build(std::vector<AggregationExprRef> expr_refs);
 
   ::substrait::Expression* makeScalarExpr(std::string func_name,
                                           std::vector<::substrait::Expression*> args,