set hive.skewjoin.key=100000; 这个是join的键对应的记录条数超过这个值则会进行优化
mapjoin
set hive.auto.current.join=true;
hive.mapjoin.smalltable.filesize默认值是25mb
select /*+mapjoin(A)*/ f.a,f.b from A t join B f on (f.a=t.a)
简单总结下,mapjoin的使用场景:
关联操作中有一张表非常小
不等值的链接操作
Bucket join
两个表以相同方式划分桶
两个表的桶个数是倍数关系
crete table order(cid int,price float) clustered by(cid) into 32 buckets;
crete table customer(id int,first string) clustered by(id) into 32 buckets;
select price from order t join customer s on t.cid=s.id
join 优化前
select m.cid,u.id from order m join customer u on m.cid=u.id where m.dt=‘2013-12-12‘;
join优化后
select m.cid,u.id from (select cid from order where dt=‘2013-12-12‘)m join customer u on m.cid=u.id;
group by 优化
hive.groupby.skewindata=true;如果是group by 过程出现倾斜 应该设置为true
set hive.groupby.mapaggr.checkinterval=100000;--这个是group的键对应的记录条数超过这个值则会进行优化
count distinct 优化
优化前
select count(distinct id) from tablename
优化后
select count(1) from (select distinct id from tablename) tmp;
select count(1) from (select id from tablename group by id) tmp;
优化前
select a,sum(b),count(distinct c),count(distinct d) from test group by a
优化后
select a,sum(b) as b,count(c) as c,count(d) as d from(select a,0 as b,c,null as d from test group by a,c union all select a,0 as b,null as c,d from test group by a,d union all select a,b,null as c,null as d from test)tmp1 group by a;
