From 3366a60239dad163e4d6ffe998df57c4f6976957 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Tue, 14 Jun 2022 19:26:02 +0900 Subject: [PATCH 01/11] Add a dummy transformation rule --- traindb-core/pom.xml | 23 ++----- .../java/traindb/planner/TrainDBPlanner.java | 4 ++ .../rules/ApproxAggregateSynopsisRule.java | 64 +++++++++++++++++++ .../traindb/planner/rules/TrainDBRules.java | 24 +++++++ traindb-project/pom.xml | 6 ++ 5 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java create mode 100644 traindb-core/src/main/java/traindb/planner/rules/TrainDBRules.java diff --git a/traindb-core/pom.xml b/traindb-core/pom.xml index f5ff8f3..7685e73 100644 --- a/traindb-core/pom.xml +++ b/traindb-core/pom.xml @@ -89,6 +89,10 @@ limitations under the License. org.apache.calcite calcite-testkit + + org.immutables + value + sqlline sqlline @@ -255,25 +259,6 @@ limitations under the License. - - org.apache.maven.plugins - maven-shade-plugin - - - - org.postgresql:postgresql - - - - - - package - - shade - - - - org.apache.maven.plugins maven-surefire-plugin diff --git a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java index 0b6d9fb..d9a024b 100644 --- a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java +++ b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.runtime.Hook; import org.checkerframework.checker.nullness.qual.Nullable; +import traindb.planner.rules.TrainDBRules; public class TrainDBPlanner { @@ -40,6 +41,9 @@ public static VolcanoPlanner createPlanner(@Nullable RelOptCostFactory costFacto @Nullable Context externalContext) { VolcanoPlanner planner = new VolcanoPlanner(costFactory, externalContext); + // TrainDB rules + planner.addRule(TrainDBRules.APPROX_AGGREGATE_SYNOPSIS); + RelOptUtil.registerDefaultRules(planner, true, Hook.ENABLE_BINDABLE.get(false)); planner.addRelTraitDef(ConventionTraitDef.INSTANCE); planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); diff --git a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java new file mode 100644 index 0000000..6601c80 --- /dev/null +++ b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package traindb.planner.rules; + +import java.util.List; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.rules.TransformationRule; +import org.immutables.value.Value; + +@Value.Enclosing +public class ApproxAggregateSynopsisRule + extends RelRule + implements TransformationRule { + + protected ApproxAggregateSynopsisRule(Config config) { + super(config); + } + + private static boolean isApproximateAggregate(Aggregate aggregate) { + List hints = aggregate.getHints(); + for (RelHint hint : hints) { + if (hint.hintName.equals("APPROXIMATE_AGGR")) { + return true; + } + } + return false; + } + + //~ Methods ---------------------------------------------------------------- + @Override public void onMatch(RelOptRuleCall call) { + // TODO replace target base table to its corresponding synopsis + return; + } + + /** Rule configuration. */ + @Value.Immutable(singleton = true) + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableApproxAggregateSynopsisRule.Config.of() + .withOperandSupplier(b -> + b.operand(LogicalAggregate.class) + .predicate(ApproxAggregateSynopsisRule::isApproximateAggregate) + .anyInputs()); + + @Override default ApproxAggregateSynopsisRule toRule() { + return new ApproxAggregateSynopsisRule(this); + } + } +} diff --git a/traindb-core/src/main/java/traindb/planner/rules/TrainDBRules.java b/traindb-core/src/main/java/traindb/planner/rules/TrainDBRules.java new file mode 100644 index 0000000..63cddd6 --- /dev/null +++ b/traindb-core/src/main/java/traindb/planner/rules/TrainDBRules.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package traindb.planner.rules; + +public class TrainDBRules { + + private TrainDBRules() { + } + + public static final ApproxAggregateSynopsisRule APPROX_AGGREGATE_SYNOPSIS = + ApproxAggregateSynopsisRule.Config.DEFAULT.toRule(); +} diff --git a/traindb-project/pom.xml b/traindb-project/pom.xml index 6378109..057a148 100644 --- a/traindb-project/pom.xml +++ b/traindb-project/pom.xml @@ -113,6 +113,12 @@ limitations under the License. avatica-core 1.20.0 + + org.immutables + value + 2.9.0 + provided + org.datanucleus From 3c7fbc0a59c4433b81333e52a0396fd324116bfc Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Fri, 17 Jun 2022 11:10:50 +0900 Subject: [PATCH 02/11] Chore: add cleanup stmts in jdbc tests --- traindb-core/src/test/java/traindb/test/JdbcClientTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/traindb-core/src/test/java/traindb/test/JdbcClientTest.java b/traindb-core/src/test/java/traindb/test/JdbcClientTest.java index 4b3ff1c..bbd30a5 100644 --- a/traindb-core/src/test/java/traindb/test/JdbcClientTest.java +++ b/traindb-core/src/test/java/traindb/test/JdbcClientTest.java @@ -184,6 +184,8 @@ public void testJdbcClientTrainModel() throws SQLException { TestUtil.printResultSet(rs); stmt.execute("DROP SYNOPSIS sales_syn"); + stmt.execute("DROP MODEL INSTANCE tgan"); + stmt.execute("DROP MODEL tablegan"); rs.close(); stmt.close(); From 01d07b56e9701eb3891fd1c408f7ca7c30486022 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Fri, 17 Jun 2022 11:24:19 +0900 Subject: [PATCH 03/11] Refactoring: pass CatalogContext to planner --- .../traindb/jdbc/TrainDBConnectionImpl.java | 4 ++ .../java/traindb/planner/TrainDBPlanner.java | 37 +++++++++---------- .../traindb/prepare/TrainDBPrepareImpl.java | 10 ++++- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/traindb-core/src/main/java/traindb/jdbc/TrainDBConnectionImpl.java b/traindb-core/src/main/java/traindb/jdbc/TrainDBConnectionImpl.java index 3a99c1c..4e6cef4 100644 --- a/traindb-core/src/main/java/traindb/jdbc/TrainDBConnectionImpl.java +++ b/traindb-core/src/main/java/traindb/jdbc/TrainDBConnectionImpl.java @@ -651,6 +651,10 @@ public static class TrainDBContextImpl implements Context { final boolean enable = config().spark(); return CalcitePrepare.Dummy.getSparkHandler(enable); } + + public CatalogContext getCatalogContext() { + return connection.getCatalogContext(); + } } /** Implementation of {@link CalciteServerStatement}. */ diff --git a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java index d9a024b..dd7f32e 100644 --- a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java +++ b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java @@ -22,35 +22,34 @@ import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.runtime.Hook; import org.checkerframework.checker.nullness.qual.Nullable; +import traindb.catalog.CatalogContext; import traindb.planner.rules.TrainDBRules; -public class TrainDBPlanner { +public class TrainDBPlanner extends VolcanoPlanner { - private TrainDBPlanner() { - } + private CatalogContext catalogContext; - public static VolcanoPlanner createPlanner() { - return createPlanner(null, null); + public TrainDBPlanner(CatalogContext catalogContext) { + this(catalogContext, null, null); } - public static VolcanoPlanner createPlanner(Context externalContext) { - return createPlanner(null, externalContext); + public TrainDBPlanner(CatalogContext catalogContext, + @Nullable RelOptCostFactory costFactory, + @Nullable Context externalContext) { + super(costFactory, externalContext); + this.catalogContext = catalogContext; + initPlanner(); } - public static VolcanoPlanner createPlanner(@Nullable RelOptCostFactory costFactory, - @Nullable Context externalContext) { - VolcanoPlanner planner = new VolcanoPlanner(costFactory, externalContext); - + public void initPlanner() { // TrainDB rules - planner.addRule(TrainDBRules.APPROX_AGGREGATE_SYNOPSIS); - - RelOptUtil.registerDefaultRules(planner, true, Hook.ENABLE_BINDABLE.get(false)); - planner.addRelTraitDef(ConventionTraitDef.INSTANCE); - planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - planner.setTopDownOpt(false); + addRule(TrainDBRules.APPROX_AGGREGATE_SYNOPSIS); - Hook.PLANNER.run(planner); // allow test to add or remove rules + RelOptUtil.registerDefaultRules(this, true, Hook.ENABLE_BINDABLE.get(false)); + addRelTraitDef(ConventionTraitDef.INSTANCE); + addRelTraitDef(RelCollationTraitDef.INSTANCE); + setTopDownOpt(false); - return planner; + Hook.PLANNER.run(this); // allow test to add or remove rules } } diff --git a/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java b/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java index 099148d..be3dcab 100644 --- a/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java +++ b/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java @@ -124,6 +124,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.Nullable; +import traindb.catalog.CatalogContext; import traindb.common.TrainDBLogger; import traindb.engine.TrainDBListResultSet; import traindb.engine.TrainDBQueryEngine; @@ -191,8 +192,11 @@ private ParseResult convert_(Context context, String sql, boolean analyze, final Convention resultConvention = enableBindable ? BindableConvention.INSTANCE : EnumerableConvention.INSTANCE; + // Use the Volcano because it can handle the traits. - final VolcanoPlanner planner = TrainDBPlanner.createPlanner(); + CatalogContext catalogContext = + ((TrainDBConnectionImpl.TrainDBContextImpl) context).getCatalogContext(); + final VolcanoPlanner planner = new TrainDBPlanner(catalogContext); final SqlToRelConverter.Config config = SqlToRelConverter.config().withTrimUnusedFields(true) @@ -414,7 +418,9 @@ protected RelOptPlanner createPlanner( if (externalContext == null) { externalContext = Contexts.of(prepareContext.config()); } - final VolcanoPlanner planner = TrainDBPlanner.createPlanner(costFactory, externalContext); + CatalogContext catalogContext = + ((TrainDBConnectionImpl.TrainDBContextImpl) prepareContext).getCatalogContext(); + final VolcanoPlanner planner = new TrainDBPlanner(catalogContext, costFactory, externalContext); return planner; } From eb098c0f019a8c74cf580598f734ba34c8af4db0 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Fri, 24 Jun 2022 10:46:58 +0900 Subject: [PATCH 04/11] Refactoring: rename method to avoid confusion --- .../src/main/java/traindb/catalog/CatalogContext.java | 2 +- .../src/main/java/traindb/catalog/JDOCatalogContext.java | 2 +- .../src/main/java/traindb/engine/TrainDBQueryEngine.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java b/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java index 94f1624..6e05163 100644 --- a/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java +++ b/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java @@ -51,7 +51,7 @@ MModelInstance trainModelInstance( MSynopsis createSynopsis(String synopsisName, String modeInstanceName) throws CatalogException; - Collection getSynopses() throws CatalogException; + Collection getAllSynopses() throws CatalogException; boolean synopsisExists(String name) throws CatalogException; diff --git a/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java b/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java index 9f9d5b6..e71feb7 100644 --- a/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java +++ b/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java @@ -181,7 +181,7 @@ public MSynopsis createSynopsis(String synopsisName, String modelInstanceName) } @Override - public Collection getSynopses() throws CatalogException { + public Collection getAllSynopses() throws CatalogException { try { Query query = pm.newQuery(MSynopsis.class); return (List) query.execute(); diff --git a/traindb-core/src/main/java/traindb/engine/TrainDBQueryEngine.java b/traindb-core/src/main/java/traindb/engine/TrainDBQueryEngine.java index 1b60ff2..07c2e54 100644 --- a/traindb-core/src/main/java/traindb/engine/TrainDBQueryEngine.java +++ b/traindb-core/src/main/java/traindb/engine/TrainDBQueryEngine.java @@ -364,7 +364,7 @@ public TrainDBListResultSet showSynopses() throws Exception { List header = Arrays.asList("synopsis", "model_instance", "schema", "table", "columns"); List> synopsisInfo = new ArrayList<>(); - for (MSynopsis mSynopsis : catalogContext.getSynopses()) { + for (MSynopsis mSynopsis : catalogContext.getAllSynopses()) { MModelInstance mModelInstance = mSynopsis.getModelInstance(); synopsisInfo.add(Arrays.asList(mSynopsis.getName(), mModelInstance.getName(), mModelInstance.getSchemaName(), mModelInstance.getTableName(), From b8471042f0fa496f1b6b36a2cde8fd9f4cc01ffe Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Fri, 24 Jun 2022 11:50:21 +0900 Subject: [PATCH 05/11] Add catalog method to get availabe synopses --- .../main/java/traindb/catalog/CatalogContext.java | 2 ++ .../java/traindb/catalog/JDOCatalogContext.java | 15 +++++++++++++++ .../main/java/traindb/planner/TrainDBPlanner.java | 12 ++++++++++++ 3 files changed, 29 insertions(+) diff --git a/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java b/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java index 6e05163..61a1823 100644 --- a/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java +++ b/traindb-catalog/src/main/java/traindb/catalog/CatalogContext.java @@ -53,6 +53,8 @@ MModelInstance trainModelInstance( Collection getAllSynopses() throws CatalogException; + Collection getAllSynopses(String baseSchema, String baseTable) throws CatalogException; + boolean synopsisExists(String name) throws CatalogException; MSynopsis getSynopsis(String name) throws CatalogException; diff --git a/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java b/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java index e71feb7..894bd6d 100644 --- a/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java +++ b/traindb-catalog/src/main/java/traindb/catalog/JDOCatalogContext.java @@ -190,6 +190,21 @@ public Collection getAllSynopses() throws CatalogException { } } + @Override + public Collection getAllSynopses(String baseSchema, String baseTable) + throws CatalogException { + try { + Query query = pm.newQuery(MSynopsis.class); + query.setFilter( + "modelInstance.schemaName == baseSchema && modelInstance.tableName == baseTable"); + query.declareParameters("String baseSchema, String baseTable"); + Collection ret = (List) query.execute(baseSchema, baseTable); + return ret; + } catch (RuntimeException e) { + throw new CatalogException("failed to get synopses", e); + } + } + @Override public boolean synopsisExists(String name) throws CatalogException { return getSynopsis(name) != null; diff --git a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java index dd7f32e..b26942f 100644 --- a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java +++ b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java @@ -14,6 +14,7 @@ package traindb.planner; +import java.util.Collection; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCostFactory; @@ -23,6 +24,8 @@ import org.apache.calcite.runtime.Hook; import org.checkerframework.checker.nullness.qual.Nullable; import traindb.catalog.CatalogContext; +import traindb.catalog.CatalogException; +import traindb.catalog.pm.MSynopsis; import traindb.planner.rules.TrainDBRules; public class TrainDBPlanner extends VolcanoPlanner { @@ -52,4 +55,13 @@ public void initPlanner() { Hook.PLANNER.run(this); // allow test to add or remove rules } + + public Collection getAvailableSynopses(String baseSchema, String baseTable) { + try { + Collection synopses = catalogContext.getAllSynopses(baseSchema, baseTable); + // TODO check columns + return synopses; + } catch (CatalogException e) {} + return null; + } } From d6c616d2905eb5d25f667be44c24905f99b17d9e Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Wed, 20 Jul 2022 18:07:08 +0900 Subject: [PATCH 06/11] Refactor: add CatalogReader to TrainDBPlanner --- .../main/java/traindb/planner/TrainDBPlanner.java | 14 ++++++++++++-- .../java/traindb/prepare/TrainDBPrepareImpl.java | 12 ++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java index b26942f..83feeac 100644 --- a/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java +++ b/traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java @@ -15,9 +15,11 @@ package traindb.planner; import java.util.Collection; +import java.util.List; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCostFactory; +import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.RelCollationTraitDef; @@ -27,20 +29,24 @@ import traindb.catalog.CatalogException; import traindb.catalog.pm.MSynopsis; import traindb.planner.rules.TrainDBRules; +import traindb.prepare.TrainDBCatalogReader; public class TrainDBPlanner extends VolcanoPlanner { private CatalogContext catalogContext; + private TrainDBCatalogReader catalogReader; - public TrainDBPlanner(CatalogContext catalogContext) { - this(catalogContext, null, null); + public TrainDBPlanner(CatalogContext catalogContext, TrainDBCatalogReader catalogReader) { + this(catalogContext, catalogReader, null, null); } public TrainDBPlanner(CatalogContext catalogContext, + TrainDBCatalogReader catalogReader, @Nullable RelOptCostFactory costFactory, @Nullable Context externalContext) { super(costFactory, externalContext); this.catalogContext = catalogContext; + this.catalogReader = catalogReader; initPlanner(); } @@ -64,4 +70,8 @@ public Collection getAvailableSynopses(String baseSchema, String base } catch (CatalogException e) {} return null; } + + public RelOptTable getTable(List names) { + return catalogReader.getTable(names); + } } diff --git a/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java b/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java index be3dcab..50208c4 100644 --- a/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java +++ b/traindb-core/src/main/java/traindb/prepare/TrainDBPrepareImpl.java @@ -196,7 +196,7 @@ private ParseResult convert_(Context context, String sql, boolean analyze, // Use the Volcano because it can handle the traits. CatalogContext catalogContext = ((TrainDBConnectionImpl.TrainDBContextImpl) context).getCatalogContext(); - final VolcanoPlanner planner = new TrainDBPlanner(catalogContext); + final VolcanoPlanner planner = new TrainDBPlanner(catalogContext, catalogReader); final SqlToRelConverter.Config config = SqlToRelConverter.config().withTrimUnusedFields(true) @@ -420,7 +420,15 @@ protected RelOptPlanner createPlanner( } CatalogContext catalogContext = ((TrainDBConnectionImpl.TrainDBContextImpl) prepareContext).getCatalogContext(); - final VolcanoPlanner planner = new TrainDBPlanner(catalogContext, costFactory, externalContext); + + TrainDBCatalogReader catalogReader = + new TrainDBCatalogReader( + prepareContext.getRootSchema(), + prepareContext.getDefaultSchemaPath(), + prepareContext.getTypeFactory(), + prepareContext.config()); + final VolcanoPlanner planner = new TrainDBPlanner( + catalogContext, catalogReader, costFactory, externalContext); return planner; } From 5423f44b7cb55b8f959dfdce112fdf32df522f8f Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Wed, 20 Jul 2022 18:18:08 +0900 Subject: [PATCH 07/11] Refactor: adjust visibility of JdbcTableScan class --- .../src/main/java/traindb/adapter/jdbc/JdbcTableScan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traindb-core/src/main/java/traindb/adapter/jdbc/JdbcTableScan.java b/traindb-core/src/main/java/traindb/adapter/jdbc/JdbcTableScan.java index 3d4977d..32c9040 100644 --- a/traindb-core/src/main/java/traindb/adapter/jdbc/JdbcTableScan.java +++ b/traindb-core/src/main/java/traindb/adapter/jdbc/JdbcTableScan.java @@ -39,7 +39,7 @@ public class JdbcTableScan extends TableScan implements JdbcRel { public final TrainDBJdbcTable jdbcTable; - protected JdbcTableScan( + public JdbcTableScan( RelOptCluster cluster, List hints, RelOptTable table, From 00d19c1d578de7a793c802c6ddd0043578e2abe0 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Wed, 20 Jul 2022 18:19:32 +0900 Subject: [PATCH 08/11] Feat: find available synopses for aggr table scan --- .../rules/ApproxAggregateSynopsisRule.java | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java index 6601c80..ee6d03b 100644 --- a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java +++ b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java @@ -14,14 +14,26 @@ package traindb.planner.rules; +import com.google.common.collect.Multimap; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.rules.TransformationRule; +import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import traindb.catalog.pm.MSynopsis; +import traindb.planner.TrainDBPlanner; @Value.Enclosing public class ApproxAggregateSynopsisRule @@ -42,10 +54,91 @@ private static boolean isApproximateAggregate(Aggregate aggregate) { return false; } + /** + * Returns a list of all table scans used by this expression or its children. + */ + private List findAllTableScans(RelNode rel) { + final Multimap, RelNode> nodes = + rel.getCluster().getMetadataQuery().getNodeTypes(rel); + final List usedTableScans = new ArrayList<>(); + if (nodes == null) { + return usedTableScans; + } + for (Map.Entry, Collection> e : nodes.asMap().entrySet()) { + if (TableScan.class.isAssignableFrom(e.getKey())) { + for (RelNode node : e.getValue()) { + usedTableScans.add((TableScan) node); + } + } + } + return usedTableScans; + } + + /** + * Returns the parent of the target node under input ancestor node. + */ + private RelNode getParent(RelNode ancestor, final RelNode target) { + if (ancestor == target) { + return null; + } + final List parentNodes = new ArrayList<>(); + new RelVisitor() { + @Override public void visit(RelNode node, int ordinal, @Nullable RelNode parent) { + if (node == target) { + parentNodes.add(parent); + } + for (RelNode input : node.getInputs()) { + if (input instanceof RelSubset) { + visit(((RelSubset) input).getBestOrOriginal(), ordinal, node); + } + else { + visit(input, ordinal, node); + } + } + } + }.go(ancestor); + if (parentNodes.size() == 1) { + return parentNodes.get(0); + } + return null; + } + //~ Methods ---------------------------------------------------------------- @Override public void onMatch(RelOptRuleCall call) { - // TODO replace target base table to its corresponding synopsis - return; + if (!(call.getPlanner() instanceof TrainDBPlanner)) { + return; + } + final TrainDBPlanner planner = (TrainDBPlanner) call.getPlanner(); + + final Aggregate aggregate = call.rel(0); + List tableScans = findAllTableScans(aggregate); + for (TableScan ts : tableScans) { + // TODO check if the tablescan node includes aggregate columns + // and does not include non-aggregate columns + + RelNode tsParent = getParent(aggregate, ts); + if (tsParent == null) { + continue; + } + List tqn = ts.getTable().getQualifiedName(); + String tableSchema = tqn.get(1); + String tableName = tqn.get(2); + + Collection candidateSynopses = + planner.getAvailableSynopses(tableSchema, tableName); + for (MSynopsis synopses : candidateSynopses) { + // TODO choose a synopsis + + List synopsisNames = new ArrayList<>(); + synopsisNames.add(tqn.get(0)); + synopsisNames.add(tqn.get(1)); + synopsisNames.add(synopses.getName()); + + RelOptTable synopsisTable = planner.getTable(synopsisNames); + + // TODO replace base table scan to synopsis table scan + } + } } /** Rule configuration. */ From 948ea11b12b3678f4f9091f0ed52a2b771c03b01 Mon Sep 17 00:00:00 2001 From: Choon Seo Park Date: Mon, 25 Jul 2022 15:42:40 +0900 Subject: [PATCH 09/11] Feat: replace base table scan to synopsis table scan --- .../rules/ApproxAggregateSynopsisRule.java | 79 ++++++++++++++----- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java index ee6d03b..9868fab 100644 --- a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java +++ b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java @@ -20,25 +20,33 @@ import java.util.List; import java.util.Map; import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelRule; import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.prepare.RelOptTableImpl; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.rules.TransformationRule; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import traindb.adapter.jdbc.JdbcConvention; +import traindb.adapter.jdbc.JdbcTableScan; +import traindb.adapter.jdbc.TrainDBJdbcTable; import traindb.catalog.pm.MSynopsis; import traindb.planner.TrainDBPlanner; @Value.Enclosing public class ApproxAggregateSynopsisRule - extends RelRule - implements TransformationRule { + extends RelRule + implements TransformationRule { protected ApproxAggregateSynopsisRule(Config config) { super(config); @@ -59,7 +67,7 @@ private static boolean isApproximateAggregate(Aggregate aggregate) { */ private List findAllTableScans(RelNode rel) { final Multimap, RelNode> nodes = - rel.getCluster().getMetadataQuery().getNodeTypes(rel); + rel.getCluster().getMetadataQuery().getNodeTypes(rel); final List usedTableScans = new ArrayList<>(); if (nodes == null) { return usedTableScans; @@ -112,31 +120,62 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { final Aggregate aggregate = call.rel(0); List tableScans = findAllTableScans(aggregate); - for (TableScan ts : tableScans) { + for (TableScan scan : tableScans) { // TODO check if the tablescan node includes aggregate columns // and does not include non-aggregate columns - RelNode tsParent = getParent(aggregate, ts); - if (tsParent == null) { + RelNode parent = getParent(aggregate, scan); + if (parent == null) { continue; } - List tqn = ts.getTable().getQualifiedName(); + if (!(parent instanceof Filter || parent instanceof Project)) { + continue; + } + List tqn = scan.getTable().getQualifiedName(); String tableSchema = tqn.get(1); String tableName = tqn.get(2); Collection candidateSynopses = - planner.getAvailableSynopses(tableSchema, tableName); - for (MSynopsis synopses : candidateSynopses) { + planner.getAvailableSynopses(tableSchema, tableName); + if (candidateSynopses == null || candidateSynopses.isEmpty()) { + continue; + } + MSynopsis bestSynopsis = null; + for (MSynopsis synopsis : candidateSynopses) { // TODO choose a synopsis + bestSynopsis = synopsis; + } - List synopsisNames = new ArrayList<>(); - synopsisNames.add(tqn.get(0)); - synopsisNames.add(tqn.get(1)); - synopsisNames.add(synopses.getName()); + List synopsisNames = new ArrayList<>(); + synopsisNames.add(tqn.get(0)); + synopsisNames.add(tqn.get(1)); + synopsisNames.add(bestSynopsis.getName()); + + RelOptTableImpl synopsisTable = (RelOptTableImpl) planner.getTable(synopsisNames); + JdbcTableScan newScan = new JdbcTableScan(scan.getCluster(), scan.getHints(), synopsisTable, + (TrainDBJdbcTable) synopsisTable.table(), (JdbcConvention) scan.getConvention()); + RelSubset subset = planner.register(newScan, null); + + if (parent instanceof Project) { + List projects = ((Project) parent).getProjects(); + List newProjects = new ArrayList<>(); + for (int i = 0; i < projects.size(); i++) { + RexInputRef inputRef = (RexInputRef) projects.get(i); + int newIndex = bestSynopsis.getModelInstance().getColumnNames() + .indexOf(parent.getRowType().getFieldNames().get(i)); + newProjects.add(new RexInputRef(newIndex, inputRef.getType())); + } - RelOptTable synopsisTable = planner.getTable(synopsisNames); + LogicalProject newProject = new LogicalProject( + parent.getCluster(), parent.getTraitSet(), ((Project) parent).getHints(), + subset, newProjects, parent.getRowType()); - // TODO replace base table scan to synopsis table scan + RelNode grandParent = getParent(aggregate, parent); + RelSubset newSubset = planner.register(newProject, null); + grandParent.replaceInput(0, newSubset); + } + else { + parent.replaceInput(0, subset); } } } @@ -145,10 +184,10 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { @Value.Immutable(singleton = true) public interface Config extends RelRule.Config { Config DEFAULT = ImmutableApproxAggregateSynopsisRule.Config.of() - .withOperandSupplier(b -> - b.operand(LogicalAggregate.class) - .predicate(ApproxAggregateSynopsisRule::isApproximateAggregate) - .anyInputs()); + .withOperandSupplier(b -> + b.operand(LogicalAggregate.class) + .predicate(ApproxAggregateSynopsisRule::isApproximateAggregate) + .anyInputs()); @Override default ApproxAggregateSynopsisRule toRule() { return new ApproxAggregateSynopsisRule(this); From 7389944f694716a8ba92a08a90c80b4a491f95e9 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Tue, 26 Jul 2022 14:10:55 +0900 Subject: [PATCH 10/11] Fix: adjust the span of approximate aggr hints --- .../rules/ApproxAggregateSynopsisRule.java | 38 +++++++++++++------ .../sql/calcite/TrainDBHintStrategyTable.java | 1 + .../traindb/sql/calcite/TrainDBSqlSelect.java | 7 +++- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java index 9868fab..909c0a4 100644 --- a/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java +++ b/traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java @@ -45,8 +45,8 @@ @Value.Enclosing public class ApproxAggregateSynopsisRule - extends RelRule - implements TransformationRule { + extends RelRule + implements TransformationRule { protected ApproxAggregateSynopsisRule(Config config) { super(config); @@ -62,12 +62,22 @@ private static boolean isApproximateAggregate(Aggregate aggregate) { return false; } + private static boolean isApproximateTableScan(TableScan scan) { + List hints = scan.getHints(); + for (RelHint hint : hints) { + if (hint.hintName.equals("APPROXIMATE_AGGR_TABLE")) { + return true; + } + } + return false; + } + /** * Returns a list of all table scans used by this expression or its children. */ private List findAllTableScans(RelNode rel) { final Multimap, RelNode> nodes = - rel.getCluster().getMetadataQuery().getNodeTypes(rel); + rel.getCluster().getMetadataQuery().getNodeTypes(rel); final List usedTableScans = new ArrayList<>(); if (nodes == null) { return usedTableScans; @@ -124,6 +134,10 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { // TODO check if the tablescan node includes aggregate columns // and does not include non-aggregate columns + if (!isApproximateTableScan(scan)) { + continue; + } + RelNode parent = getParent(aggregate, scan); if (parent == null) { continue; @@ -136,7 +150,7 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { String tableName = tqn.get(2); Collection candidateSynopses = - planner.getAvailableSynopses(tableSchema, tableName); + planner.getAvailableSynopses(tableSchema, tableName); if (candidateSynopses == null || candidateSynopses.isEmpty()) { continue; } @@ -153,7 +167,7 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { RelOptTableImpl synopsisTable = (RelOptTableImpl) planner.getTable(synopsisNames); JdbcTableScan newScan = new JdbcTableScan(scan.getCluster(), scan.getHints(), synopsisTable, - (TrainDBJdbcTable) synopsisTable.table(), (JdbcConvention) scan.getConvention()); + (TrainDBJdbcTable) synopsisTable.table(), (JdbcConvention) scan.getConvention()); RelSubset subset = planner.register(newScan, null); if (parent instanceof Project) { @@ -162,13 +176,13 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { for (int i = 0; i < projects.size(); i++) { RexInputRef inputRef = (RexInputRef) projects.get(i); int newIndex = bestSynopsis.getModelInstance().getColumnNames() - .indexOf(parent.getRowType().getFieldNames().get(i)); + .indexOf(parent.getRowType().getFieldNames().get(i)); newProjects.add(new RexInputRef(newIndex, inputRef.getType())); } LogicalProject newProject = new LogicalProject( - parent.getCluster(), parent.getTraitSet(), ((Project) parent).getHints(), - subset, newProjects, parent.getRowType()); + parent.getCluster(), parent.getTraitSet(), ((Project) parent).getHints(), + subset, newProjects, parent.getRowType()); RelNode grandParent = getParent(aggregate, parent); RelSubset newSubset = planner.register(newProject, null); @@ -184,10 +198,10 @@ private RelNode getParent(RelNode ancestor, final RelNode target) { @Value.Immutable(singleton = true) public interface Config extends RelRule.Config { Config DEFAULT = ImmutableApproxAggregateSynopsisRule.Config.of() - .withOperandSupplier(b -> - b.operand(LogicalAggregate.class) - .predicate(ApproxAggregateSynopsisRule::isApproximateAggregate) - .anyInputs()); + .withOperandSupplier(b -> + b.operand(LogicalAggregate.class) + .predicate(ApproxAggregateSynopsisRule::isApproximateAggregate) + .anyInputs()); @Override default ApproxAggregateSynopsisRule toRule() { return new ApproxAggregateSynopsisRule(this); diff --git a/traindb-core/src/main/java/traindb/sql/calcite/TrainDBHintStrategyTable.java b/traindb-core/src/main/java/traindb/sql/calcite/TrainDBHintStrategyTable.java index f5d7de8..79bab5b 100644 --- a/traindb-core/src/main/java/traindb/sql/calcite/TrainDBHintStrategyTable.java +++ b/traindb-core/src/main/java/traindb/sql/calcite/TrainDBHintStrategyTable.java @@ -27,6 +27,7 @@ private static HintStrategyTable createHintStrategies() { static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) { return builder .hintStrategy("APPROXIMATE_AGGR", HintPredicates.AGGREGATE) + .hintStrategy("APPROXIMATE_AGGR_TABLE", HintPredicates.TABLE_SCAN) .build(); } } diff --git a/traindb-core/src/main/java/traindb/sql/calcite/TrainDBSqlSelect.java b/traindb-core/src/main/java/traindb/sql/calcite/TrainDBSqlSelect.java index c41853f..70146c4 100644 --- a/traindb-core/src/main/java/traindb/sql/calcite/TrainDBSqlSelect.java +++ b/traindb-core/src/main/java/traindb/sql/calcite/TrainDBSqlSelect.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSelect; -import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.parser.Span; import org.apache.calcite.sql.parser.SqlParserPos; import org.checkerframework.checker.nullness.qual.Nullable; @@ -57,6 +57,11 @@ public TrainDBSqlSelect(SqlParserPos pos, hintList.add(new SqlHint(pos, new SqlIdentifier("APPROXIMATE_AGGR", pos), SqlNodeList.EMPTY, SqlHint.HintOptionFormat.EMPTY)); setHints(new SqlNodeList(hintList, pos)); + + SqlParserPos fromPos = Span.of(from).end(from); + hintList.add(new SqlHint(fromPos, new SqlIdentifier("APPROXIMATE_AGGR_TABLE", fromPos), + SqlNodeList.EMPTY, SqlHint.HintOptionFormat.EMPTY)); + setHints(new SqlNodeList(hintList, fromPos)); } } From 3eccb084521d5983e5100d8dc931c1ae17b15273 Mon Sep 17 00:00:00 2001 From: Taewhi Lee Date: Tue, 26 Jul 2022 14:12:30 +0900 Subject: [PATCH 11/11] Test: add test cases for SELECT APPROXIMATE query --- traindb-core/src/test/resources/sql/basic.iq | 100 +--------------- .../src/test/resources/sql/synopsis.iq | 110 ++++++++++++++++++ 2 files changed, 111 insertions(+), 99 deletions(-) create mode 100644 traindb-core/src/test/resources/sql/synopsis.iq diff --git a/traindb-core/src/test/resources/sql/basic.iq b/traindb-core/src/test/resources/sql/basic.iq index ca16359..ed409f5 100644 --- a/traindb-core/src/test/resources/sql/basic.iq +++ b/traindb-core/src/test/resources/sql/basic.iq @@ -74,102 +74,4 @@ WHERE a.product_id = b.product_id; +---------+ (1 row) -!ok - -CREATE MODEL tablegan TYPE SYNOPSIS LOCAL AS 'TableGAN' IN 'models/TableGAN.py'; -(0 rows modified) - -!update - -SHOW MODELS; -+----------+----------+----------+----------+--------------------+ -| model | type | location | class | uri | -+----------+----------+----------+----------+--------------------+ -| tablegan | SYNOPSIS | LOCAL | TableGAN | models/TableGAN.py | -+----------+----------+----------+----------+--------------------+ -(1 row) - -!ok - -TRAIN MODEL tablegan INSTANCE tgan ON instacart_small.order_products(product_id, add_to_cart_order); -(0 rows modified) - -!update - -SHOW MODEL INSTANCES; -+----------+----------------+-----------------+----------------+---------------------------------+ -| model | model_instance | schema | table | columns | -+----------+----------------+-----------------+----------------+---------------------------------+ -| tablegan | tgan | instacart_small | order_products | [product_id, add_to_cart_order] | -+----------+----------------+-----------------+----------------+---------------------------------+ -(1 row) - -!ok - -CREATE SYNOPSIS order_products_syn FROM MODEL INSTANCE tgan LIMIT 1000; -(0 rows modified) - -!update - -SHOW SYNOPSES; -+--------------------+----------------+-----------------+----------------+---------------------------------+ -| synopsis | model_instance | schema | table | columns | -+--------------------+----------------+-----------------+----------------+---------------------------------+ -| order_products_syn | tgan | instacart_small | order_products | [product_id, add_to_cart_order] | -+--------------------+----------------+-----------------+----------------+---------------------------------+ -(1 row) - -!ok - -SELECT count(*) as c2 FROM instacart_small.order_products_syn; -+------+ -| c2 | -+------+ -| 1000 | -+------+ -(1 row) - -!ok - -DROP SYNOPSIS order_products_syn; -(0 rows modified) - -!update - -SHOW SYNOPSES; -+ - | -+ -+ -(0 rows) - -!ok - -DROP MODEL INSTANCE tgan; -(0 rows modified) - -!update - -SHOW MODEL INSTANCES; -+ - | -+ -+ -(0 rows) - -!ok - -DROP MODEL tablegan; -(0 rows modified) - -!update - -SHOW MODELS; -+ - | -+ -+ -(0 rows) - -!ok - +!ok \ No newline at end of file diff --git a/traindb-core/src/test/resources/sql/synopsis.iq b/traindb-core/src/test/resources/sql/synopsis.iq new file mode 100644 index 0000000..f115c9f --- /dev/null +++ b/traindb-core/src/test/resources/sql/synopsis.iq @@ -0,0 +1,110 @@ +# synopsis.iq +!use instacart_small +!set outputformat mysql +!set maxwidth 300 + +CREATE MODEL tablegan TYPE SYNOPSIS LOCAL AS 'TableGAN' IN 'models/TableGAN.py'; +(0 rows modified) + +!update + +SHOW MODELS; ++----------+----------+----------+----------+--------------------+ +| model | type | location | class | uri | ++----------+----------+----------+----------+--------------------+ +| tablegan | SYNOPSIS | LOCAL | TableGAN | models/TableGAN.py | ++----------+----------+----------+----------+--------------------+ +(1 row) + +!ok + +TRAIN MODEL tablegan INSTANCE tgan ON instacart_small.order_products(reordered, add_to_cart_order); +(0 rows modified) + +!update + +SHOW MODEL INSTANCES; ++----------+----------------+-----------------+----------------+--------------------------------+ +| model | model_instance | schema | table | columns | ++----------+----------------+-----------------+----------------+--------------------------------+ +| tablegan | tgan | instacart_small | order_products | [reordered, add_to_cart_order] | ++----------+----------------+-----------------+----------------+--------------------------------+ +(1 row) + +!ok + +CREATE SYNOPSIS order_products_syn FROM MODEL INSTANCE tgan LIMIT 1000; +(0 rows modified) + +!update + +SHOW SYNOPSES; ++--------------------+----------------+-----------------+----------------+--------------------------------+ +| synopsis | model_instance | schema | table | columns | ++--------------------+----------------+-----------------+----------------+--------------------------------+ +| order_products_syn | tgan | instacart_small | order_products | [reordered, add_to_cart_order] | ++--------------------+----------------+-----------------+----------------+--------------------------------+ +(1 row) + +!ok + +SELECT count(*) as c2 FROM instacart_small.order_products_syn; ++------+ +| c2 | ++------+ +| 1000 | ++------+ +(1 row) + +!ok + +SELECT APPROXIMATE avg(add_to_cart_order) as average FROM instacart_small.order_products; + +JdbcToEnumerableConverter + JdbcProject(average=[CAST(/(CASE(=($1, 0), null:INTEGER, $0), $1)):INTEGER]) + JdbcAggregate(group=[{}], agg#0=[$SUM0($1)], agg#1=[COUNT()]) + JdbcTableScan(table=[[traindb, instacart_small, order_products_syn]]) +!plan + +DROP SYNOPSIS order_products_syn; +(0 rows modified) + +!update + +SHOW SYNOPSES; ++ + | ++ ++ +(0 rows) + +!ok + +DROP MODEL INSTANCE tgan; +(0 rows modified) + +!update + +SHOW MODEL INSTANCES; ++ + | ++ ++ +(0 rows) + +!ok + +DROP MODEL tablegan; +(0 rows modified) + +!update + +SHOW MODELS; ++ + | ++ ++ +(0 rows) + +!ok +