diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java index 3999a6ce9b64..52c8d4cd5663 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java @@ -110,6 +110,9 @@ public interface CalciteConnectionConfig extends ConnectionConfig { boolean lenientOperatorLookup(); /** Returns the value of {@link CalciteConnectionProperty#TOPDOWN_OPT}. */ boolean topDownOpt(); + /** Returns the value of + * {@link CalciteConnectionProperty#TOPDOWN_GENERAL_DECORRELATION_ENABLED}. */ + boolean topDownGeneralDecorrelationEnabled(); /** Returns the value of {@link CalciteConnectionProperty#META_TABLE_FACTORY}, * or a default meta table factory if not set. If diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java index 7d9f9f76d1cc..221259cef959 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java @@ -215,6 +215,11 @@ public boolean isSet(CalciteConnectionProperty property) { .getBoolean(); } + @Override public boolean topDownGeneralDecorrelationEnabled() { + return CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED.wrap(properties) + .getBoolean(); + } + @Override public @PolyNull T metaTableFactory( Class metaTableFactoryClass, @PolyNull T defaultMetaTableFactory) { diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java index a9c7627025cd..32079be2cfbc 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java @@ -154,6 +154,9 @@ public enum CalciteConnectionProperty implements ConnectionProperty { * If true (the default), Calcite de-correlates the plan. */ FORCE_DECORRELATE("forceDecorrelate", Type.BOOLEAN, true, false), + TOPDOWN_GENERAL_DECORRELATION_ENABLED("topDownGeneralDecorrelationEnabled", + Type.BOOLEAN, false, false), + /** Type system. The name of a class that implements * {@link org.apache.calcite.rel.type.RelDataTypeSystem} and has a public * default constructor or an {@code INSTANCE} constant. */ diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 6bb21e1ae2af..8439fabd5f3c 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -107,8 +107,10 @@ import org.apache.calcite.sql2rel.SqlRexConvertletTable; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.sql2rel.StandardConvertletTable; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -1091,6 +1093,9 @@ private PreparedResult prepare_(Supplier fn, SqlValidator validator, CatalogReader catalogReader, SqlToRelConverter.Config config) { + config = + config.withTopDownGeneralDecorrelationEnabled( + context.config().topDownGeneralDecorrelationEnabled()); return new SqlToRelConverter(this, validator, catalogReader, cluster, convertletTable, config); } @@ -1107,6 +1112,11 @@ private PreparedResult prepare_(Supplier fn, @Override protected RelNode decorrelate(SqlToRelConverter sqlToRelConverter, SqlNode query, RelNode rootRel) { + if (context.config().topDownGeneralDecorrelationEnabled()) { + final RelBuilder relBuilder = + sqlToRelConverter.config().getRelBuilderFactory().create(rootRel.getCluster(), null); + return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder); + } return sqlToRelConverter.decorrelate(query, rootRel); } diff --git a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java index ff2903047d03..5fece8fa9770 100644 --- a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java @@ -51,6 +51,7 @@ import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.sql2rel.SqlRexConvertletTable; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; @@ -262,8 +263,10 @@ private void ready() { final RelOptCluster cluster = RelOptCluster.create(requireNonNull(planner, "planner"), rexBuilder); - final SqlToRelConverter.Config config = - sqlToRelConverterConfig.withTrimUnusedFields(false); + final SqlToRelConverter.Config config = sqlToRelConverterConfig + .withTrimUnusedFields(false) + .withTopDownGeneralDecorrelationEnabled( + connectionConfig.topDownGeneralDecorrelationEnabled()); final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(this, validator, createCatalogReader(), cluster, convertletTable, config); @@ -272,8 +275,9 @@ private void ready() { root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); - root = - root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); + root = config.isTopDownGeneralDecorrelationEnabled() + ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder)) + : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); state = State.STATE_5_CONVERTED; return root; } @@ -314,20 +318,22 @@ public class ViewExpanderImpl implements ViewExpander { final RexBuilder rexBuilder = createRexBuilder(); final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); - final SqlToRelConverter.Config config = - sqlToRelConverterConfig.withTrimUnusedFields(false); + final SqlToRelConverter.Config config = sqlToRelConverterConfig + .withTrimUnusedFields(false) + .withTopDownGeneralDecorrelationEnabled( + connectionConfig.topDownGeneralDecorrelationEnabled()); final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(this, validator, catalogReader, cluster, convertletTable, config); - final RelRoot root = + RelRoot root = sqlToRelConverter.convertQuery(sqlNode, true, false); - final RelRoot root2 = - root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); + root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); - return root2.withRel( - RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); + return config.isTopDownGeneralDecorrelationEnabled() + ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder)) + : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); } // CalciteCatalogReader is stateless; no need to store one diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index 726c07bb142b..3e3cf46a1117 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -175,11 +175,10 @@ protected FilterProjectTransposeRule( final RelNode input = project.getInput(); final RelTraitSet traitSet = filter.getTraitSet() .replaceIfs(RelCollationTraitDef.INSTANCE, - () -> Collections.singletonList( - input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE))) + () -> input.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE)) .replaceIfs(RelDistributionTraitDef.INSTANCE, () -> Collections.singletonList( - input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE))); + input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE))); newCondition = RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition); newFilterRel = filter.copy(traitSet, input, newCondition); } else { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index da96b98b0655..dd97b2a8fdbf 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3966,6 +3966,9 @@ protected boolean enableDecorrelation() { } protected RelNode decorrelateQuery(RelNode rootRel) { + if (config.isTopDownGeneralDecorrelationEnabled()) { + return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder); + } return RelDecorrelator.decorrelateQuery(rootRel, relBuilder); } @@ -6496,6 +6499,14 @@ public interface Config { /** Sets {@link #isDecorrelationEnabled()}. */ Config withDecorrelationEnabled(boolean decorrelationEnabled); + /** Returns whether to use the top-down general decorrelator. */ + @Value.Default default boolean isTopDownGeneralDecorrelationEnabled() { + return false; + } + + /** Sets {@link #isTopDownGeneralDecorrelationEnabled()}. */ + Config withTopDownGeneralDecorrelationEnabled(boolean topDownGeneralDecorrelationEnabled); + /** Returns the {@code trimUnusedFields} option. Controls whether to trim * unused fields as part of the conversion process. */ @Value.Default default boolean isTrimUnusedFields() { diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index af6d62a5cb14..f9e5933f3ffc 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -46,6 +46,7 @@ import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.sql2rel.RelFieldTrimmer; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; @@ -259,7 +260,26 @@ public static Program subQuery(RelMetadataProvider metadataProvider) { CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); - return of(builder.build(), true, metadataProvider); + final Program oldProgram = of(builder.build(), true, metadataProvider); + + final HepProgramBuilder newBuilder = HepProgram.builder(); + newBuilder.addRuleCollection( + ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); + final Program newProgram = of(newBuilder.build(), true, metadataProvider); + + return (planner, rel, requiredOutputTraits, materializations, lattices) -> { + final CalciteConnectionConfig config = + planner.getContext().maybeUnwrap(CalciteConnectionConfig.class) + .orElse(CalciteConnectionConfig.DEFAULT); + final Program program = config.topDownGeneralDecorrelationEnabled() + ? newProgram + : oldProgram; + return program.run(planner, rel, requiredOutputTraits, materializations, + lattices); + }; } public static Program measure(RelMetadataProvider metadataProvider) { @@ -425,6 +445,9 @@ private static class DecorrelateProgram implements Program { if (config.forceDecorrelate()) { final RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null); + if (config.topDownGeneralDecorrelationEnabled()) { + return TopDownGeneralDecorrelator.decorrelateQuery(rel, relBuilder); + } return RelDecorrelator.decorrelateQuery(rel, relBuilder); } return rel; diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index 1aa6ab187137..b878bfb085e0 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -41,7 +41,7 @@ /** * Test that runs every Quidem file in the "core" module as a test. */ -class CoreQuidemTest extends QuidemTest { +public class CoreQuidemTest extends QuidemTest { /** Runs a test from the command line. * *

For example: @@ -57,6 +57,12 @@ public static void main(String[] args) throws Exception { /** For {@link QuidemTest#test(String)} parameters. */ @Override public Collection getPath() { + return data(); + } + + /** Returns the list of Quidem files to run. + * Subclasses can override this method to gradually add files. */ + protected Collection data() { // Start with a test file we know exists, then find the directory and list // its files. final String first = "sql/agg.iq"; @@ -68,31 +74,31 @@ public static void main(String[] args) throws Exception { @Override public Connection connect(String name, boolean reference) throws Exception { switch (name) { case "blank": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") - .with(CalciteAssert.SchemaSpec.BLANK) + .with(CalciteAssert.SchemaSpec.BLANK)) .connect(); case "scott": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-spark": discard(CustomTypeSystems.SPARK_TYPE_SYSTEM); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#SPARK_TYPE_SYSTEM") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-checked-rounding-half-up": discard(CustomTypeSystems.ROUNDING_MODE_HALF_UP); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") // Use bigquery conformance, which forces checked arithmetic @@ -100,84 +106,84 @@ public static void main(String[] args) throws Exception { .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#ROUNDING_MODE_HALF_UP") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-negative-scale": discard(CustomTypeSystems.NEGATIVE_SCALE); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-negative-scale-rounding-half-up": discard(CustomTypeSystems.NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-lenient": // Same as "scott", but uses LENIENT conformance. // TODO: add a way to change conformance without defining a new // connection - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.LENIENT) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-babel": // Same as "scott", but uses BABEL conformance. // connection - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.BABEL) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-mysql": // Same as "scott", but uses MySQL conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.MYSQL_5) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-oracle": // Same as "scott", but uses Oracle conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.ORACLE_10) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-mssql": // Same as "scott", but uses SQL_SERVER_2008 conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.SQL_SERVER_2008) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "steelwheels": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteAssert.SchemaSpec.STEELWHEELS) - .with(Lex.BIG_QUERY) + .with(Lex.BIG_QUERY)) .connect(); default: return super.connect(name, reference); diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java new file mode 100644 index 000000000000..406bd3bc8f34 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.config.CalciteConnectionProperty; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Test that runs Quidem files with the top-down decorrelator enabled. + */ +public class CoreQuidemTest2 extends CoreQuidemTest { + /** Runs a test from the command line. + * + *

For example: + * + *

+ * java CoreQuidemTest2 sql/dummy.iq + *
*/ + public static void main(String[] args) throws Exception { + for (String arg : args) { + new CoreQuidemTest2().test(arg); + } + } + + @Override protected Collection data() { + final List paths = new ArrayList<>(super.data()); + // These remove operations are temporary and will be deleted + // once the new decorrelator can adapt to all scenarios. + + // TODO: The following files involves UNNEST and LEFT_MARK JOIN + paths.remove("sql/agg.iq"); + paths.remove("sql/measure.iq"); + paths.remove("sql/unnest.iq"); + paths.remove("sql/lateral.iq"); + paths.remove("sql/some.iq"); + paths.remove("sql/sub-query.iq"); + paths.remove("sql/scalar.iq"); + paths.remove("sql/join.iq"); + paths.remove("sql/spatial.iq"); + paths.remove("sql/measure-paper.iq"); + paths.remove("sql/misc.iq"); + return paths; + } + + @Override protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat assertThat) { + return super.customize(assertThat) + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, true); + } + + @Override protected boolean useTopDownGeneralDecorrelator() { + return true; + } +} diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index 9c200caf7e2f..28f78c8e9230 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -89,6 +89,7 @@ insert into table2 values (NULL, 1), (2, 1); # Checked on Oracle !set lateDecorrelate true select i, j from table1 where table1.j NOT IN (select i from table2 where table1.i=table2.j); +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[IS NULL($t1)], expr#11=[IS NOT NULL($t7)], expr#12=[<($t4, $t3)], expr#13=[OR($t10, $t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[OR($t9, $t14)], proj#0..1=[{exprs}], $condition=[$t15]) EnumerableMergeJoin(condition=[AND(=($0, $6), =($1, $5))], joinType=[left]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -113,7 +114,15 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[ (0 rows) !ok +!} +# TODO: This error needs to be fixed +!if (use_new_decorr) { +Unable to convert LEFT_MARK to Linq4j JoinType +!error +!} + +!if (use_old_decorr) { select * from table1 where j not in (select i from table2); +---+---+ | I | J | @@ -153,5 +162,12 @@ select * from table1 where j not in (select i from table2) or j = 3; (1 row) !ok +!} + +# TODO: This error needs to be fixed +!if (use_new_decorr) { +Unable to convert LEFT_MARK to Linq4j JoinType +!error +!} # End blank.iq diff --git a/core/src/test/resources/sql/conditions.iq b/core/src/test/resources/sql/conditions.iq index 5fa94d08891e..7ce0c78c9c20 100644 --- a/core/src/test/resources/sql/conditions.iq +++ b/core/src/test/resources/sql/conditions.iq @@ -418,6 +418,7 @@ where empno = 7369; !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10]) @@ -430,6 +431,22 @@ EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableAggregate(group=[{}], agg#0=[COUNT()]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], EMPNO=[$t0]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], DUMMY=[$t2]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[0:BIGINT], expr#2=[>($t0, $t1)], $f0=[$t0], $condition=[$t2]) + EnumerableAggregate(group=[{}], agg#0=[COUNT()]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} # sub-query return true with Equal condition select r.empno, s.deptno diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index baa57592fa90..556d10f721af 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -141,6 +141,8 @@ WHERE e1.mgr > 12 (5 rows) !ok + +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2]) EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t10]) @@ -149,6 +151,17 @@ EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t3)], expr#12=[AND($t10, $t11)], MGR=[$t3], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..2=[{inputs}], MGR=[$t1], COMM=[$t2]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $4)], joinType=[semi]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t3)], expr#12=[AND($t10, $t11)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t12]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} !set hep-rules original # End hep.iq diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 0461cc644b76..1147a534eff5 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -359,6 +359,7 @@ or !ok +!if (use_old_decorr) { EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))], joinType=[inner]) EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) @@ -366,6 +367,16 @@ EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)], expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) !plan +!} + +!if (use_new_decorr) { +EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) + EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) + EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) + EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) +!plan +!} !set planner-rules original # [CALCITE-7086] Implement a rule that performs the inverse operation of AggregateCaseToFilterRule diff --git a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java index c12618e13aec..8d744d829bee 100644 --- a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java +++ b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java @@ -25,6 +25,7 @@ import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.plan.Contexts; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -207,6 +208,7 @@ protected static void populate(SqlIdentifier name, SqlNode query, requireNonNull( Schemas.subSchema(context.getRootSchema(), context.getDefaultSchemaPath())).plus()) + .context(Contexts.of(context.config())) .build(); final Planner planner = Frameworks.getPlanner(config); try { diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index d4ec9d2ba9dc..9564089b7bfe 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -144,10 +144,18 @@ public static class ExplainValidatedCommand extends AbstractCommand { private static final Pattern PATTERN = Pattern.compile("\\.iq$"); // Saved original planner rules - private static @Nullable List originalRules; + private @Nullable List originalRules; - private static @Nullable Object getEnv(String varName) { + protected boolean useTopDownGeneralDecorrelator() { + return false; + } + + private @Nullable Object getEnv(String varName) { switch (varName) { + case "use_old_decorr": + return !useTopDownGeneralDecorrelator(); + case "use_new_decorr": + return useTopDownGeneralDecorrelator(); case "jdk18": return System.getProperty("java.version").startsWith("1.8"); case "fixed": @@ -199,7 +207,7 @@ protected static Collection data(String first) { final List paths = new ArrayList<>(); final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$"); for (File f : Util.first(dir.listFiles(filter), new File[0])) { - paths.add(f.getAbsolutePath().substring(commonPrefixLength)); + paths.add(n2u(f.getAbsolutePath().substring(commonPrefixLength))); } return paths; } @@ -220,7 +228,8 @@ protected void checkRun(String path) throws Exception { // outFile = "/home/fred/calcite/core/build/quidem/test/sql/agg.iq" final URL inUrl = QuidemTest.class.getResource("/" + n2u(path)); inFile = Sources.of(requireNonNull(inUrl, "inUrl")).file(); - outFile = replaceDir(inFile, "resources", "quidem"); + outFile = replaceDir(inFile, "resources", "quidem/" + + getClass().getSimpleName()); } Util.discard(outFile.getParentFile().mkdirs()); try (Reader reader = Util.reader(inFile); @@ -254,7 +263,7 @@ protected void checkRun(String path) throws Exception { // - Reset defaults: "original" if (propertyName.equals("planner-rules")) { if (value.equals("original")) { - closer.add(Hook.PLANNER.addThread(QuidemTest::resetPlanner)); + closer.add(Hook.PLANNER.addThread(this::resetPlanner)); } else { closer.add( Hook.PLANNER.addThread((Consumer) @@ -302,7 +311,7 @@ protected void checkRun(String path) throws Exception { } } }) - .withEnv(QuidemTest::getEnv) + .withEnv(this::getEnv) .build(); new Quidem(config).execute(); } @@ -325,7 +334,7 @@ private static void updatePlanner(RelOptPlanner planner, String value) { rulesAdd.forEach(planner::addRule); } - private static void resetPlanner(RelOptPlanner planner) { + private void resetPlanner(RelOptPlanner planner) { if (originalRules != null) { planner.getRules().forEach(planner::removeRule); originalRules.forEach(planner::addRule); @@ -445,6 +454,11 @@ private static File replaceDir(File file, String target, String replacement) { n2u('/' + replacement + '/'))); } + /** Allows subclasses to customize the connection. */ + protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat assertThat) { + return assertThat; + } + /** Creates a command handler. */ protected CommandHandler createCommandHandler() { return Quidem.EMPTY_COMMAND_HANDLER; @@ -489,7 +503,7 @@ public void test(String path) throws Exception { protected abstract Collection getPath(); /** Quidem connection factory for Calcite's built-in test schemas. */ - protected static class QuidemConnectionFactory + protected class QuidemConnectionFactory implements Quidem.ConnectionFactory { public Connection connect(String name) throws Exception { return connect(name, false); @@ -511,89 +525,89 @@ public Connection connect(String name) throws Exception { } switch (name) { case "hr": - return CalciteAssert.hr() + return customize(CalciteAssert.hr()) .connect(); case "aux": - return CalciteAssert.hr() - .with(CalciteAssert.Config.AUX) + return customize(CalciteAssert.hr() + .with(CalciteAssert.Config.AUX)) .connect(); case "foodmart": - return CalciteAssert.that() - .with(CalciteAssert.Config.FOODMART_CLONE) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.FOODMART_CLONE)) .connect(); case "geo": - return CalciteAssert.that() - .with(CalciteAssert.Config.GEO) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.GEO)) .connect(); case "scott": - return CalciteAssert.that() - .with(CalciteAssert.Config.SCOTT) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.SCOTT)) .connect(); case "jdbc_scott": - return CalciteAssert.that() - .with(CalciteAssert.Config.JDBC_SCOTT) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.JDBC_SCOTT)) .connect(); case "steelwheels": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.STEELWHEELS) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.STEELWHEELS)) .connect(); case "jdbc_steelwheels": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS)) .connect(); case "post": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "post-postgresql": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "standard,postgresql") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "post-big-query": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "standard,bigquery") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "mysqlfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "mysql") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "sparkfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "spark") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "oraclefunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "oracle") - .with(CalciteAssert.Config.REGULAR) + .with(CalciteAssert.Config.REGULAR)) .connect(); case "mssqlfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "mssql") - .with(CalciteAssert.Config.REGULAR) + .with(CalciteAssert.Config.REGULAR)) .connect(); case "catchall": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.TIME_ZONE, "UTC") .withSchema("s", new ReflectiveSchemaWithoutRowCount( - new CatchallSchema())) + new CatchallSchema()))) .connect(); case "orinoco": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.ORINOCO) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.ORINOCO)) .connect(); case "seq": - final Connection connection = CalciteAssert.that() - .withSchema("s", new AbstractSchema()) + final Connection connection = customize(CalciteAssert.that() + .withSchema("s", new AbstractSchema())) .connect(); connection.unwrap(CalciteConnection.class).getRootSchema() .subSchemas().get("s") @@ -611,8 +625,8 @@ public Connection connect(String name) throws Exception { }); return connection; case "bookstore": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.BOOKSTORE) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.BOOKSTORE)) .connect(); default: throw new RuntimeException("unknown connection '" + name + "'");