Skip to content

HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter #6757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
soumyakanti3578 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter #6757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d911f98
HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter
soumyakanti3578 Sep 3, 2026
77d5854
Accept plan changes where extra Project is removed
soumyakanti3578 Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...c/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private static void convertOpTree(RelNode rel, RelNode parent) {
introduceDerivedTable(rel.getInput(0), rel);
}
} else if (rel instanceof SingleRel) {
if (rel instanceof HiveJdbcConverter) {
if (rel instanceof HiveJdbcConverter && !(parent instanceof Project)) {
introduceDerivedTable(rel, parent);
} else if (rel instanceof Filter) {
if (!validFilterParent(rel, parent)) {
Expand Down
50 changes: 50 additions & 0 deletions ql/src/test/queries/clientpositive/jdbc_redundant_project.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
--! qt:database:postgres:qdb:q_test_country_table_with_schema.postgres.sql

CREATE EXTERNAL TABLE country_bob (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}",
"hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}",
"hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}",
"hive.sql.schema" = "bob",
"hive.sql.table" = "country");

CREATE EXTERNAL TABLE country_alice (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}",
"hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}",
"hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}",
"hive.sql.schema" = "alice",
"hive.sql.table" = "country");

EXPLAIN CBO
WITH cross_items AS (
SELECT id
FROM (
SELECT id FROM country_bob
INTERSECT
SELECT id FROM country_alice
INTERSECT
SELECT id FROM country_bob
) x
)
SELECT channel, cnt
FROM (
SELECT 'a' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'b' channel, COUNT(*) cnt
FROM country_alice
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'c' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
) y
ORDER BY channel;
23 changes: 11 additions & 12 deletions ql/src/test/results/clientpositive/llap/cbo_jdbc_joincost.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,15 @@ POSTHOOK: Input: default@book
#### A masked pattern was here ####
CBO PLAN:
HiveProject(a.lname=[$0], b.title=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
HiveProject(lname=[$0], title=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
HiveJdbcConverter(convention=[JDBC.MYSQL]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(lname=[$1], title=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcJoin(condition=[=($0, $3)], joinType=[inner]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(id=[$0], lname=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcFilter(condition=[IS NOT NULL($0)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(id=[$0], lname=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcHiveTableScan(table=[[default, author]], table:alias=[a]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(title=[$0], author=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcFilter(condition=[IS NOT NULL($1)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(title=[$1], author=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcHiveTableScan(table=[[default, book]], table:alias=[b]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
HiveJdbcConverter(convention=[JDBC.MYSQL]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(lname=[$1], title=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcJoin(condition=[=($0, $3)], joinType=[inner]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(id=[$0], lname=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcFilter(condition=[IS NOT NULL($0)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(id=[$0], lname=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcHiveTableScan(table=[[default, author]], table:alias=[a]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(title=[$0], author=[$1]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcFilter(condition=[IS NOT NULL($1)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcProject(title=[$1], author=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
JdbcHiveTableScan(table=[[default, book]], table:alias=[b]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###

55 changes: 27 additions & 28 deletions ql/src/test/results/clientpositive/llap/external_jdbc_rowcount.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,32 +266,31 @@ POSTHOOK: Input: default@ext_simple_derby_table14
#### A masked pattern was here ####
CBO PLAN:
HiveProject(t1.fkey=[$0], t2.ikey=[$1], _c2=[$2])
HiveProject(fkey=[$0], ikey=[$1], _o__c2=[$2])
HiveJdbcConverter(convention=[JDBC.DERBY])
JdbcProject(fkey=[$0], ikey=[$2], _o__c2=[$3])
JdbcAggregate(group=[{0, 2, 4}], agg#0=[sum($1)])
JdbcFilter(condition=[IS NULL($3)])
JdbcJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[left])
JdbcProject(fkey=[$1], ikey=[$0])
JdbcAggregate(group=[{0, 1}])
JdbcFilter(condition=[IS NULL($4)])
JdbcJoin(condition=[=($0, $3)], joinType=[left])
JdbcProject(ikey=[$0], fkey=[$2], dkey=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table11]], table:alias=[ext_simple_derby_table11])
JdbcProject(ikey2=[$0], fkey2=[$1], dkey2=[$2])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey2=[$0], fkey2=[$2], dkey2=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table13]], table:alias=[ext_simple_derby_table13])
JdbcProject(datekey=[$2], fkey=[$1], ikey=[$0])
JdbcAggregate(group=[{0, 1, 3}])
JdbcFilter(condition=[IS NULL($5)])
JdbcJoin(condition=[=($0, $4)], joinType=[left])
JdbcProject(ikey=[$0], fkey=[$1], dkey=[$2], datekey=[$3])
JdbcFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))])
JdbcProject(ikey=[$0], fkey=[$2], dkey=[$3], datekey=[$4])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table12]], table:alias=[ext_simple_derby_table12])
JdbcProject(ikey2=[$0], fkey2=[$1], dkey2=[$2])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey2=[$0], fkey2=[$2], dkey2=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table14]], table:alias=[ext_simple_derby_table14])
HiveJdbcConverter(convention=[JDBC.DERBY])
JdbcProject(fkey=[$0], ikey=[$2], _o__c2=[$3])
JdbcAggregate(group=[{0, 2, 4}], agg#0=[sum($1)])
JdbcFilter(condition=[IS NULL($3)])
JdbcJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[left])
JdbcProject(fkey=[$1], ikey=[$0])
JdbcAggregate(group=[{0, 1}])
JdbcFilter(condition=[IS NULL($4)])
JdbcJoin(condition=[=($0, $3)], joinType=[left])
JdbcProject(ikey=[$0], fkey=[$2], dkey=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table11]], table:alias=[ext_simple_derby_table11])
JdbcProject(ikey2=[$0], fkey2=[$1], dkey2=[$2])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey2=[$0], fkey2=[$2], dkey2=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table13]], table:alias=[ext_simple_derby_table13])
JdbcProject(datekey=[$2], fkey=[$1], ikey=[$0])
JdbcAggregate(group=[{0, 1, 3}])
JdbcFilter(condition=[IS NULL($5)])
JdbcJoin(condition=[=($0, $4)], joinType=[left])
JdbcProject(ikey=[$0], fkey=[$1], dkey=[$2], datekey=[$3])
JdbcFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))])
JdbcProject(ikey=[$0], fkey=[$2], dkey=[$3], datekey=[$4])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table12]], table:alias=[ext_simple_derby_table12])
JdbcProject(ikey2=[$0], fkey2=[$1], dkey2=[$2])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey2=[$0], fkey2=[$2], dkey2=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table14]], table:alias=[ext_simple_derby_table14])

23 changes: 11 additions & 12 deletions ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,16 +574,15 @@ POSTHOOK: Input: default@ext_simple_derby_table2
#### A masked pattern was here ####
CBO PLAN:
HiveProject(ext_simple_derby_table1.fkey=[$0], ext_simple_derby_table2.dkey=[$1])
HiveProject(fkey=[$0], dkey=[$1])
HiveJdbcConverter(convention=[JDBC.DERBY])
JdbcProject(fkey=[$1], dkey=[$3])
JdbcJoin(condition=[=($0, $2)], joinType=[inner])
JdbcProject(ikey=[$0], fkey=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey=[$0], fkey=[$2])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table1]], table:alias=[ext_simple_derby_table1])
JdbcProject(ikey=[$0], dkey=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey=[$0], dkey=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table2]], table:alias=[ext_simple_derby_table2])
HiveJdbcConverter(convention=[JDBC.DERBY])
JdbcProject(fkey=[$1], dkey=[$3])
JdbcJoin(condition=[=($0, $2)], joinType=[inner])
JdbcProject(ikey=[$0], fkey=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey=[$0], fkey=[$2])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table1]], table:alias=[ext_simple_derby_table1])
JdbcProject(ikey=[$0], dkey=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(ikey=[$0], dkey=[$3])
JdbcHiveTableScan(table=[[default, ext_simple_derby_table2]], table:alias=[ext_simple_derby_table2])

14 changes: 5 additions & 9 deletions ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7292,16 +7292,12 @@ GROUP BY "ss_store_sk"
Map-reduce partition columns: ss_store_sk (type: int)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE
top n: 6
Select Operator
expressions: ss_store_sk (type: int), $f1 (type: decimal(17,2))
outputColumnNames: _col0, _col1
Reduce Output Operator
key expressions: ss_store_sk (type: int), $f1 (type: decimal(17,2))
null sort order: az
sort order: ++
Map-reduce partition columns: ss_store_sk (type: int)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: decimal(17,2))
null sort order: az
sort order: ++
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: no inputs
Reducer 2
Expand Down
7 changes: 3 additions & 4 deletions ql/src/test/results/clientpositive/llap/jdbc_filter_expand_row_operator.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ POSTHOOK: Input: default@book
#### A masked pattern was here ####
CBO PLAN:
HiveProject(book.id=[$0], book.title=[$1], book.author=[$2])
HiveProject(id=[$0], title=[$1], author=[$2])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcFilter(condition=[AND(OR(=($0, 0), AND(=($0, 1), =($2, 11)), AND(=($0, 2), =($2, 22))), IN($0, 0, 1, 2))])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcFilter(condition=[AND(OR(=($0, 0), AND(=($0, 1), =($2, 11)), AND(=($0, 2), =($2, 22))), IN($0, 0, 1, 2))])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])

80 changes: 38 additions & 42 deletions ql/src/test/results/clientpositive/llap/jdbc_project_pushdown.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,11 @@ POSTHOOK: Input: default@book
#### A masked pattern was here ####
CBO PLAN:
HiveProject(id=[$0])
HiveProject(id=[$0])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcProject(id=[$0])
JdbcFilter(condition=[=(SUBSTRING($1, 0, 4), 'Les')])
JdbcProject(id=[$0], title=[$1])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcProject(id=[$0])
JdbcFilter(condition=[=(SUBSTRING($1, 0, 4), 'Les')])
JdbcProject(id=[$0], title=[$1])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])

PREHOOK: query: explain select id from book where substring(title, 0, 4) = 'Les'
PREHOOK: type: QUERY
Expand Down Expand Up @@ -198,18 +197,17 @@ POSTHOOK: Input: default@book
#### A masked pattern was here ####
CBO PLAN:
HiveProject(book.title=[$0], author.fname=[$1])
HiveProject(title=[$0], fname=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcProject(title=[$0], fname=[$3])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcProject(title=[$0], fname=[$3])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])

PREHOOK: query: explain select book.title, author.fname from book join author on book.author = author.id
PREHOOK: type: QUERY
Expand Down Expand Up @@ -441,18 +439,17 @@ POSTHOOK: Input: default@book
#### A masked pattern was here ####
CBO PLAN:
HiveProject(author.fname=[$0], books=[$1])
HiveProject(fname=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{3}], agg#0=[count($0)])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{3}], agg#0=[count($0)])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])

PREHOOK: query: explain
select author.fname, count(book.title) as books
Expand Down Expand Up @@ -545,18 +542,17 @@ CBO PLAN:
HiveProject(_c0=[$0], books=[$1])
HiveSortLimit(sort0=[$2], dir0=[ASC], fetch=[5])
HiveProject(_o__c0=[ucase($0)], books=[$1], _o__col4=[ucase($0)])
HiveProject(fname=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{3}], agg#0=[count($0)])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{3}], agg#0=[count($0)])
JdbcJoin(condition=[=($1, $2)], joinType=[inner])
JdbcProject(title=[$0], author=[$1])
JdbcFilter(condition=[IS NOT NULL($1)])
JdbcProject(title=[$1], author=[$2])
JdbcHiveTableScan(table=[[default, book]], table:alias=[book])
JdbcProject(id=[$0], fname=[$1])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0], fname=[$1])
JdbcHiveTableScan(table=[[default, author]], table:alias=[author])

PREHOOK: query: explain
select ucase(author.fname), count(book.title) as books
Expand Down
Loading
Loading