Hive SQL综合案例

drop database if exists movie;
create database if not exists movie;
use movie;

create table t_user(
userid bigint,
sex string,
age int,
occupation string,
zipcode string) 
row format serde ‘org.apache.hadoop.hive.serde2.RegexSerDe‘ 
with serdeproperties(‘input.regex‘=‘(.*)::(.*)::(.*)::(.*)::(.*)‘,‘output.format.string‘=‘%1$s %2$s %3$s %4$s %5$s‘)
stored as textfile;

use movie;
create table t_movie(
movieid bigint,
moviename string,
movietype string) 
row format serde ‘org.apache.hadoop.hive.serde2.RegexSerDe‘ 
with serdeproperties(‘input.regex‘=‘(.*)::(.*)::(.*)‘,‘output.format.string‘=‘%1$s %2$s %3$s‘)
stored as textfile;

use movie;
create table t_rating(
userid bigint,
movieid bigint,
rate double,
times string) 
row format serde ‘org.apache.hadoop.hive.serde2.RegexSerDe‘ 
with serdeproperties(‘input.regex‘=‘(.*)::(.*)::(.*)::(.*)‘,‘output.format.string‘=‘%1$s %2$s %3$s %4$s‘)
stored as textfile;

0: jdbc:hive2://hadoop3:10000> load data local inpath "/home/hadoop/movie/users.dat" into table t_user;
No rows affected (0.928 seconds)
0: jdbc:hive2://hadoop3:10000> load data local inpath "/home/hadoop/movie/movies.dat" into table t_movie;
No rows affected (0.538 seconds)
0: jdbc:hive2://hadoop3:10000> load data local inpath "/home/hadoop/movie/ratings.dat" into table t_rating;
No rows affected (0.963 seconds)
0: jdbc:hive2://hadoop3:10000> 

select t.* from t_user t;

select t.* from t_movie t;

select t.* from t_rating t;

create table answer2 as 
select a.moviename as moviename,count(a.moviename) as total 
from t_movie a join t_rating b on a.movieid=b.movieid 
group by a.moviename 
order by total desc 
limit 10;

select * from answer2;

create table answer3_F as 
select "F" as sex, c.moviename as name, avg(a.rate) as avgrate, count(c.moviename) as total  
from t_rating a 
join t_user b on a.userid=b.userid 
join t_movie c on a.movieid=c.movieid 
where b.sex="F" 
group by c.moviename 
having total >= 50
order by avgrate desc 
limit 10;

select * from answer3_F；

create table answer3_M as 
select "M" as sex, c.moviename as name, avg(a.rate) as avgrate, count(c.moviename) as total  
from t_rating a 
join t_user b on a.userid=b.userid 
join t_movie c on a.movieid=c.movieid 
where b.sex="M" 
group by c.moviename 
having total >= 50
order by avgrate desc 
limit 10;

select * from answer3_M；

create table answer4 as 
select a.age as age, avg(b.rate) as avgrate 
from t_user a join t_rating b on a.userid=b.userid 
where b.movieid=2116 
group by a.age;

select * from answer4;

select a.userid, count(a.userid) as total 
from t_rating a join t_user b on a.userid = b.userid 
where b.sex="F" 
group by a.userid 
order by total desc 
limit 1;

create table answer5_B as 
select a.movieid as movieid, a.rate as rate  
from t_rating a 
where a.userid=1150 
order by rate desc 
limit 10;

select * from answer5_B;

create table answer5_C as 
select b.movieid as movieid, c.moviename as moviename, avg(b.rate) as avgrate 
from answer5_B a 
join t_rating b on a.movieid=b.movieid 
join t_movie c on b.movieid=c.movieid 
group by b.movieid,c.moviename;

select * from answer5_C;

create table answer6_A as
select  a.movieid as movieid, a.moviename as moviename, substr(a.moviename,-5,4) as years, avg(b.rate) as avgrate
from t_movie a join t_rating b on a.movieid=b.movieid 
group by a.movieid, a.moviename;

select * from answer6_A;

select years, count(years) as total 
from answer6_A a 
where avgrate >= 4.0 
group by years 
order by total desc 
limit 1;

create table answer6_C as
select a.moviename as name, a.avgrate as rate 
from answer6_A a 
where a.years=1998 
order by rate desc 
limit 10;

select * from answer6_C;

create table answer7_A as 
select b.movieid as id, b.moviename as name, b.years as years, b.avgrate as rate, a.movietype as type 
from t_movie a join answer6_A b on a.movieid=b.movieid;

select t.* from answer7_A t;

create table answer7_B as 
select t.id as id, t.name as name, t.rate as rate 
from answer7_A t 
where t.years=1997 and instr(lcase(t.type),‘comedy‘) >0 
order by rate desc
limit 10;

select * from answer7_B;

create table answer8_A as 
select a.id as id, a.name as name, a.years as years, a.rate as rate, tv.type as type 
from answer7_A a 
lateral view explode(split(a.type,"\\|")) tv as type;

select * from answer8_A; 

create table answer8_B as 
select id,name,years,rate,type,row_number() over(distribute by type sort by rate desc ) as num
from answer8_A;

select * from answer8_B;

select a.* from answer8_B a where a.num <= 5;

create table answer9_A as 
select a.years as years, a.type as type, avg(a.rate) as rate 
from answer8_A a 
group by a.years,a.type 
order by rate desc;

select * from answer9_A;

create table answer9_B as 
select years,type,rate,row_number() over (distribute by years sort by rate) as num
from answer9_A;

select * from answer9_B;

select * from answer9_B where num=1;

create table answer10_A as
select c.movieid, c.moviename, avg(b.rate) as avgrate, a.zipcode
from t_user a 
join t_rating b on a.userid=b.userid 
join t_movie c on b.movieid=c.movieid 
group by a.zipcode,c.movieid, c.moviename;

select t.* from answer10_A t;

create table answer10_B as
select movieid,moviename,avgrate,zipcode, row_number() over (distribute by zipcode sort by avgrate) as num 
from answer10_A; 

select t.* from answer10_B t;

insert overwrite directory "/movie/answer10/" select t.* from answer10_B t where t.num=1;

用户名，月份，访问次数

A,2015-01,5
A,2015-01,15
B,2015-01,5
A,2015-01,8
B,2015-01,25
A,2015-01,5
A,2015-02,4
A,2015-02,6
B,2015-02,10
B,2015-02,5
A,2015-03,16
A,2015-03,22
B,2015-03,23
B,2015-03,10
B,2015-03,1

use myhive;
create external table if not exists t_access(
uname string comment ‘用户名‘,
umonth string comment ‘月份‘,
ucount int comment ‘访问次数‘
) comment ‘用户访问表‘ 
row format delimited fields terminated by "," 
location "/hive/t_access"; 

load data local inpath "/home/hadoop/access.txt" into table t_access;

select * from t_access;

--求当月访问次数
create table tmp_access(
name string,
mon string,
num int
); 

insert into table tmp_access 
select uname,umonth,sum(ucount)
 from t_access t group by t.uname,t.umonth;

select * from tmp_access;

create view tmp_view as 
select a.name anme,a.mon amon,a.num anum,b.name bname,b.mon bmon,b.num bnum from tmp_access a join tmp_access b 
on a.name=b.name;

select * from tmp_view;

select anme,amon,anum,max(bnum) as max_access,sum(bnum) as sum_access 
from tmp_view 
where amon>=bmon 
group by anme,amon,anum;

use myhive;
CREATE TABLE `course` (
  `id` int,
  `sid` int ,
  `course` string,
  `score` int 
) ;

// 插入数据
// 字段解释：id, 学号， 课程， 成绩
INSERT INTO `course` VALUES (1, 1, ‘yuwen‘, 43);
INSERT INTO `course` VALUES (2, 1, ‘shuxue‘, 55);
INSERT INTO `course` VALUES (3, 2, ‘yuwen‘, 77);
INSERT INTO `course` VALUES (4, 2, ‘shuxue‘, 88);
INSERT INTO `course` VALUES (5, 3, ‘yuwen‘, 98);
INSERT INTO `course` VALUES (6, 3, ‘shuxue‘, 65);

create view tmp_course_view as
select sid, case course when "shuxue" then score else 0 end  as shuxue,  
case course when "yuwen" then score else 0 end  as yuwen from course;  

select * from tmp_course_view;

create view tmp_course_view1 as
select aa.sid, max(aa.shuxue) as shuxue, max(aa.yuwen) as yuwen from tmp_course_view aa group by sid;  

select * from tmp_course_view1;

select * from tmp_course_view1 where shuxue > yuwen;

2010012325

2010012325表示在2010年01月23日的气温为25度

create table tmp_weather as 
select substr(data,1,4) years,substr(data,5,2) months,substr(data,7,2) days,substr(data,9,2) temp from weather;

select * from tmp_weather;

create table tmp_year_weather as 
select substr(data,1,4) years,max(substr(data,9,2)) max_temp from weather group by substr(data,1,4);

select * from tmp_year_weather;

select * from tmp_year_weather a join tmp_weather b on a.years=b.years and a.max_temp=b.temp;

id course 
1,a 
1,b 
1,c 
1,e 
2,a 
2,c 
2,d 
2,f 
3,a 
3,b 
3,c 
3,e

create table t_course(id int,course string)
row format delimited fields terminated by ",";

load data local inpath "/home/hadoop/course/course.txt" into table t_course;

id    a    b    c    d    e    f
1     1    1    1    0    1    0
2     1    0    1    1    0    1
3     1    1    1    0    1    0

select collect_set(course) as courses from id_course;

set hive.strict.checks.cartesian.product=false;

create table id_courses as select t1.id as id,t1.course as id_courses,t2.course courses 
from 
( select id as id,collect_set(course) as course from id_course group by id ) t1 
join 
(select collect_set(course) as course from id_course) t2;

select id,
case when array_contains(id_courses, courses[0]) then 1 else 0 end as a,
case when array_contains(id_courses, courses[1]) then 1 else 0 end as b,
case when array_contains(id_courses, courses[2]) then 1 else 0 end as c,
case when array_contains(id_courses, courses[3]) then 1 else 0 end as d,
case when array_contains(id_courses, courses[4]) then 1 else 0 end as e,
case when array_contains(id_courses, courses[5]) then 1 else 0 end as f 
from id_courses;

a,01,150
a,01,200
b,01,1000
b,01,800
c,01,250
c,01,220
b,01,6000
a,02,2000
a,02,3000
b,02,1000
b,02,1500
c,02,350
c,02,280
a,03,350
a,03,250

use class;
create table t_store(
name string,
months int,
money int
) 
row format delimited fields terminated by ",";

load data local inpath "/home/hadoop/store.txt" into table t_store;

create table tmp_store1 as 
select name,months,sum(money) as money from t_store group by name,months;

select * from tmp_store1;

create table tmp_store2 as 
select a.name aname,a.months amonths,a.money amoney,b.name bname,b.months bmonths,b.money bmoney from tmp_store1 a 
join tmp_store1 b on a.name=b.name order by aname,amonths;

select * from tmp_store2;

select aname,amonths,amoney,sum(bmoney) as total from tmp_store2 where amonths >= bmonths group by aname,amonths,amoney;