Add fileNameGenerator to the constructor of IcebergInsertTableHandle

.github/workflows/linux-build-base.yml

@@ -257,7 +257,7 @@ jobs:
         env:
           VELOX_DEPENDENCY_SOURCE: BUNDLED
           ICU_SOURCE: SYSTEM
-          MAKEFLAGS: NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=2
+          MAKEFLAGS: NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3
           EXTRA_CMAKE_FLAGS: >-
             -DCMAKE_LINK_LIBRARIES_STRATEGY=REORDER_FREELY
             -DVELOX_ENABLE_PARQUET=ON

.github/workflows/linux-build.yml

@@ -15,21 +15,6 @@
 name: Linux Build using GCC
 
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - velox/**
-      - '!velox/docs/**'
-      - CMakeLists.txt
-      - CMake/**
-      - scripts/setup-ubuntu.sh
-      - scripts/setup-common.sh
-      - scripts/setup-versions.sh
-      - scripts/setup-helper-functions.sh
-      - .github/workflows/linux-build.yml
-      - .github/workflows/linux-build-base.yml
-
   pull_request:
     paths:
       - velox/**

scripts/checks/license-header.py

@@ -120,7 +120,12 @@ def wrapper_hash(header, args):
 )
 
 file_pattern = regex.compile(
-    "|".join(["^" + fnmatch.translate(type) + "$" for type in file_types.keys()])
+    "|".join(
+        [
+            "^" + fnmatch.translate(type).replace(r"\z", r"\Z") + "$"
+            for type in file_types.keys()
+        ]
+    )
 )
 
 

velox/common/caching/AsyncDataCache.h

@@ -445,7 +445,7 @@ class CoalescedLoad {
     return state_;
   }
 
-  void cancel() {
+  virtual void cancel() {
     setEndState(State::kCancelled);
   }
 

velox/connectors/hive/CMakeLists.txt

@@ -15,8 +15,6 @@
 velox_add_library(velox_hive_config OBJECT HiveConfig.cpp)
 velox_link_libraries(velox_hive_config velox_core velox_exception)
 
-add_subdirectory(iceberg)
-
 velox_add_library(
   velox_hive_connector
   OBJECT
@@ -28,15 +26,17 @@ velox_add_library(
   HiveConnectorSplit.cpp
   HiveDataSink.cpp
   HiveDataSource.cpp
-  HivePartitionName.cpp
+  HivePartitionUtil.cpp
   PartitionIdGenerator.cpp
   SplitReader.cpp
   TableHandle.cpp
 )
 
+add_subdirectory(iceberg)
+
 velox_link_libraries(
   velox_hive_connector
-  PUBLIC velox_hive_iceberg_splitreader
+  PRIVATE velox_hive_iceberg_splitreader
   PRIVATE velox_common_io velox_connector velox_dwio_catalog_fbhive velox_hive_partition_function
 )
 

velox/connectors/hive/HiveConfig.cpp

@@ -273,4 +273,9 @@ std::string HiveConfig::schema(const config::ConfigBase* session) const {
       kSchema, config_->get<std::string>(kSchema, ""));
 }
 
+bool HiveConfig::fanoutEnabled(const config::ConfigBase* session) const {
+  return session->get<bool>(
+      kFanoutEnabledSession, config_->get<bool>(kFanoutEnabled, true));
+}
+
 } // namespace facebook::velox::connector::hive

velox/connectors/hive/HiveConfig.h

@@ -208,6 +208,12 @@ class HiveConfig {
   static constexpr const char* kSource = "source";
   static constexpr const char* kSchema = "schema";
 
+  /// Controls the writer mode, whether the fanout mode writer is enabled,
+  /// default value is true, setting to false means clustered mode.
+  /// Currently applies only to the Iceberg writer.
+  static constexpr const char* kFanoutEnabled = "fanout-enabled";
+  static constexpr const char* kFanoutEnabledSession = "fanout_enabled";
+
   InsertExistingPartitionsBehavior insertExistingPartitionsBehavior(
       const config::ConfigBase* session) const;
 
@@ -305,6 +311,9 @@ class HiveConfig {
   /// Schema of the query. Used for storage logging.
   std::string schema(const config::ConfigBase* session) const;
 
+  /// Return if fanout writer mode is enabled.
+  bool fanoutEnabled(const config::ConfigBase* session) const;
+
   HiveConfig(std::shared_ptr<const config::ConfigBase> config) {
     VELOX_CHECK_NOT_NULL(
         config, "Config is null for HiveConfig initialization");

velox/connectors/hive/HiveConnectorUtil.cpp

@@ -24,6 +24,10 @@
 #include "velox/expression/ExprConstants.h"
 #include "velox/expression/ExprToSubfieldFilter.h"
 
+#include <boost/lexical_cast.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
 namespace facebook::velox::connector::hive {
 namespace {
 
@@ -807,7 +811,6 @@ std::unique_ptr<dwio::common::BufferedInput> createBufferedInput(
 }
 
 namespace {
-
 core::CallTypedExprPtr replaceInputs(
     const core::CallTypedExpr* call,
     std::vector<core::TypedExprPtr>&& inputs) {
@@ -989,6 +992,10 @@ core::TypedExprPtr extractFiltersFromRemainingFilter(
 }
 } // namespace
 
+std::string makeUuid() {
+  return boost::lexical_cast<std::string>(boost::uuids::random_generator()());
+}
+
 core::TypedExprPtr extractFiltersFromRemainingFilter(
     const core::TypedExprPtr& expr,
     core::ExpressionEvaluator* evaluator,

velox/connectors/hive/HiveConnectorUtil.h

@@ -152,4 +152,6 @@ core::TypedExprPtr extractFiltersFromRemainingFilter(
     common::SubfieldFilters& filters,
     double& sampleRate);
 
+std::string makeUuid();
+
 } // namespace facebook::velox::connector::hive