Skip to content

feat: add an option to skip validate lance table #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hfutatzhanghb wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: add an option to skip validate lance table #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5d87373
feat: add an option to skip validate lance table.
Oct 13, 2025
e9c921f
add getter method for clientPool field for lance-spark.
Oct 13, 2025
f349c4d
fix format violations.
Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions ...lance-namespace-hive2/src/main/java/com/lancedb/lance/namespace/hive2/Hive2Namespace.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ public void tableExists(TableExistsRequest request) {
CommonUtil.formatCurrentStackTrace());
}

Hive2Util.validateLanceTable(hmsTable.get());
Hive2Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
}

@Override
Expand Down Expand Up @@ -461,7 +461,7 @@ protected Optional<String> doDescribeTable(ObjectIdentifier id) {
return Optional.empty();
}

Hive2Util.validateLanceTable(hmsTable.get());
Hive2Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
return Optional.of(hmsTable.get().getSd().getLocation());
}

Expand Down Expand Up @@ -594,7 +594,7 @@ protected String doDropTable(ObjectIdentifier id) {
CommonUtil.formatCurrentStackTrace());
}

Hive2Util.validateLanceTable(hmsTable.get());
Hive2Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
String location = hmsTable.get().getSd().getLocation();

clientPool.run(
Expand Down Expand Up @@ -729,4 +729,8 @@ private String getDefaultTableLocation(String namespaceName, String tableName) {
return String.format(
"%s/%s/%s.lance", config.getRoot(), namespaceName.toLowerCase(), tableName.toLowerCase());
}

public Hive2ClientPool getClientPool() {
return clientPool;
}
}
11 changes: 11 additions & 0 deletions ...namespace-hive2/src/main/java/com/lancedb/lance/namespace/hive2/Hive2NamespaceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,13 @@ public class Hive2NamespaceConfig {

public static final String ROOT_DEFAULT = System.getProperty("user.dir");

public static final String SKIP_LANCE_TABLE_VALIDATION = "skip.validate.lance.format.table";
public static final boolean SKIP_LANCE_TABLE_VALIDATION_DEFAULT = false;

private final int clientPoolSize;
private final Map<String, String> storageOptions;
private final String root;
private final boolean skipValidationLanceFormatTable;

public Hive2NamespaceConfig(Map<String, String> properties) {
this.clientPoolSize =
Expand All @@ -47,6 +51,9 @@ public Hive2NamespaceConfig(Map<String, String> properties) {
this.root =
OpenDalUtil.stripTrailingSlash(
PropertyUtil.propertyAsString(properties, ROOT, ROOT_DEFAULT));
this.skipValidationLanceFormatTable =
PropertyUtil.propertyAsBoolean(
properties, SKIP_LANCE_TABLE_VALIDATION, SKIP_LANCE_TABLE_VALIDATION_DEFAULT);
}

public int getClientPoolSize() {
Expand All @@ -60,4 +67,8 @@ public Map<String, String> getStorageOptions() {
public String getRoot() {
return root;
}

public boolean isSkipValidationLanceFormatTable() {
return skipValidationLanceFormatTable;
}
}
11 changes: 11 additions & 0 deletions java/lance-namespace-hive2/src/main/java/com/lancedb/lance/namespace/hive2/Hive2Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
Expand All @@ -34,6 +36,7 @@
import static com.lancedb.lance.namespace.hive2.Hive2ErrorType.InvalidLanceTable;

public class Hive2Util {
private static final Logger LOG = LoggerFactory.getLogger(Hive2Util.class);

public static Database getDatabaseOrNull(Hive2ClientPool clientPool, String db) {
try {
Expand Down Expand Up @@ -109,6 +112,14 @@ public static Optional<Table> getTable(Hive2ClientPool clientPool, String db, St
}

public static void validateLanceTable(Table table) {
validateLanceTable(table, false);
}

public static void validateLanceTable(Table table, boolean skipValidation) {
if (skipValidation) {
LOG.info("Skip validate lance format table procedure.");
return;
}
Map<String, String> params = table.getParameters();
if (params == null || !"lance".equalsIgnoreCase(params.get("table_type"))) {
throw LanceNamespaceException.badRequest(
Expand Down
4 changes: 3 additions & 1 deletion ...space-hive2/src/test/java/com/lancedb/lance/namespace/hive2/TestHive2NamespaceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,13 @@ public void testOtherConfigurationValues() {
ImmutableMap.of(
"root", "/custom/root",
"client.pool-size", "5",
"storage.s3.region", "us-west-2");
"storage.s3.region", "us-west-2",
"skip.validate.lance.format.table", "true");
Hive2NamespaceConfig config = new Hive2NamespaceConfig(properties);

assertEquals("/custom/root", config.getRoot());
assertEquals(5, config.getClientPoolSize());
assertEquals("us-west-2", config.getStorageOptions().get("s3.region"));
assertEquals(true, config.isSkipValidationLanceFormatTable());
}
}
10 changes: 7 additions & 3 deletions ...lance-namespace-hive3/src/main/java/com/lancedb/lance/namespace/hive3/Hive3Namespace.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ public void tableExists(TableExistsRequest request) {
CommonUtil.formatCurrentStackTrace());
}

Hive3Util.validateLanceTable(hmsTable.get());
Hive3Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
}

@Override
Expand Down Expand Up @@ -550,7 +550,7 @@ protected Optional<String> doDescribeTable(ObjectIdentifier id) {
return Optional.empty();
}

Hive3Util.validateLanceTable(hmsTable.get());
Hive3Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
return Optional.of(hmsTable.get().getSd().getLocation());
}

Expand Down Expand Up @@ -691,7 +691,7 @@ protected String doDropTable(ObjectIdentifier id) {
CommonUtil.formatCurrentStackTrace());
}

Hive3Util.validateLanceTable(hmsTable.get());
Hive3Util.validateLanceTable(hmsTable.get(), config.isSkipValidationLanceFormatTable());
String location = hmsTable.get().getSd().getLocation();

clientPool.run(
Expand Down Expand Up @@ -915,4 +915,8 @@ private String getDefaultTableLocation(String namespaceName, String tableName) {
return String.format(
"%s/%s/%s.lance", config.getRoot(), namespaceName.toLowerCase(), tableName.toLowerCase());
}

public Hive3ClientPool getClientPool() {
return clientPool;
}
}
11 changes: 11 additions & 0 deletions ...namespace-hive3/src/main/java/com/lancedb/lance/namespace/hive3/Hive3NamespaceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,13 @@ public class Hive3NamespaceConfig {

public static final String ROOT_DEFAULT = System.getProperty("user.dir");

public static final String SKIP_LANCE_TABLE_VALIDATION = "skip.validate.lance.format.table";
public static final boolean SKIP_LANCE_TABLE_VALIDATION_DEFAULT = false;

private final int clientPoolSize;
private final Map<String, String> storageOptions;
private final String root;
private final boolean skipValidationLanceFormatTable;

public Hive3NamespaceConfig(Map<String, String> properties) {
this.clientPoolSize =
Expand All @@ -48,6 +52,9 @@ public Hive3NamespaceConfig(Map<String, String> properties) {
this.root =
OpenDalUtil.stripTrailingSlash(
PropertyUtil.propertyAsString(properties, ROOT, ROOT_DEFAULT));
this.skipValidationLanceFormatTable =
PropertyUtil.propertyAsBoolean(
properties, SKIP_LANCE_TABLE_VALIDATION, SKIP_LANCE_TABLE_VALIDATION_DEFAULT);
}

public int getClientPoolSize() {
Expand All @@ -61,4 +68,8 @@ public Map<String, String> getStorageOptions() {
public String getRoot() {
return root;
}

public boolean isSkipValidationLanceFormatTable() {
return skipValidationLanceFormatTable;
}
}
12 changes: 12 additions & 0 deletions java/lance-namespace-hive3/src/main/java/com/lancedb/lance/namespace/hive3/Hive3Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
Expand All @@ -36,6 +38,8 @@
import static com.lancedb.lance.namespace.hive3.Hive3ErrorType.UnknownCatalog;

public class Hive3Util {
private static final Logger LOG = LoggerFactory.getLogger(Hive3Util.class);

public static Catalog getCatalogOrNull(Hive3ClientPool clientPool, String catalog) {
try {
return clientPool.run(client -> client.getCatalog(catalog));
Expand Down Expand Up @@ -172,6 +176,14 @@ public static Optional<Table> getTable(
}

public static void validateLanceTable(Table table) {
validateLanceTable(table, false);
}

public static void validateLanceTable(Table table, boolean skipValidation) {
if (skipValidation) {
LOG.info("Skip validate lance format table procedure.");
return;
}
Map<String, String> params = table.getParameters();
if (params == null || !"lance".equalsIgnoreCase(params.get("table_type"))) {
throw LanceNamespaceException.badRequest(
Expand Down
4 changes: 3 additions & 1 deletion ...space-hive3/src/test/java/com/lancedb/lance/namespace/hive3/TestHive3NamespaceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,13 @@ public void testOtherConfigurationValues() {
ImmutableMap.of(
"root", "/custom/root",
"client.pool-size", "5",
"storage.s3.region", "us-west-2");
"storage.s3.region", "us-west-2",
"skip.validate.lance.format.table", "true");
Hive3NamespaceConfig config = new Hive3NamespaceConfig(properties);

assertEquals("/custom/root", config.getRoot());
assertEquals(5, config.getClientPoolSize());
assertEquals("us-west-2", config.getStorageOptions().get("s3.region"));
assertEquals(true, config.isSkipValidationLanceFormatTable());
}
}