数仓1.4 |业务数仓搭建| 拉链表| Presto

id                订单编号
total_amount    订单金额
order_status    订单状态
user_id            用户id
payment_way    支付方式
out_trade_no    支付流水号
create_time    创建时间
operate_time    操作时间
        

id            订单编号
order_id    订单号
user_id    用户id
sku_id    商品id
sku_name    商品名称
order_price    商品价格
sku_num            商品数量
create_time    创建时间

id            skuId
spu_id    spuid
price        价格
sku_name    商品名称
sku_desc    商品描述
weight    重量
tm_id    品牌id
category3_id    品类id
create_time    创建时间

id            用户id
name    姓名
birthday    生日
gender    性别
email    邮箱
user_level        用户等级
create_time    创建时间

id            id
name    名称

id            id
name    名称
category1_id    一级品类id

id            id
name    名称
Category2_id    二级品类id

id                    编号
out_trade_no    对外业务编号
order_id            订单编号
user_id            用户编号
alipay_trade_no    支付宝交易流水编号
total_amount    支付金额
subject            交易内容
payment_type    支付类型
payment_time    支付时间

#!/bin/bash

db_date=$2
echo $db_date
db_name=gmall

import_data() {
/opt/module/sqoop/bin/sqoop import --connect jdbc:mysql://hadoop101:3306/$db_name \
--username root --password 123456 --target-dir  /origin_data/$db_name/db/$1/$db_date --delete-target-dir --num-mappers 1 --fields-terminated-by "\t" --query   "$2"‘ and  $CONDITIONS;‘
}

import_sku_info(){
  import_data  "sku_info"  "select 
id, spu_id, price, sku_name, sku_desc, weight, tm_id,
category3_id, create_time 
  from sku_info  where 1=1"
}

import_user_info(){
  import_data "user_info" "select 
id, name, birthday, gender, email, user_level, 
create_time 
from user_info where 1=1"
}

import_base_category1(){
  import_data "base_category1" "select 
id, name from base_category1 where 1=1"
}

import_base_category2(){
  import_data "base_category2" "select 
id, name, category1_id from base_category2 where 1=1"
}

import_base_category3(){
  import_data "base_category3" "select id, name, category2_id from base_category3 where 1=1"
}

import_order_detail(){
  import_data   "order_detail"   "select 
    od.id, 
    order_id, 
    user_id, 
    sku_id, 
    sku_name, 
    order_price, 
    sku_num, 
    o.create_time  
  from order_info o , order_detail od 
  where o.id=od.order_id 
  and DATE_FORMAT(create_time,‘%Y-%m-%d‘)=‘$db_date‘"
}

import_payment_info(){
  import_data  "payment_info"   "select 
    id,  
    out_trade_no, 
    order_id, 
    user_id, 
    alipay_trade_no, 
    total_amount,  
    subject , 
    payment_type, 
    payment_time 
  from payment_info 
  where DATE_FORMAT(payment_time,‘%Y-%m-%d‘)=‘$db_date‘"
}

import_order_info(){
  import_data   "order_info"   "select 
    id, 
    total_amount, 
    order_status, 
    user_id, 
    payment_way, 
    out_trade_no, 
    create_time, 
    operate_time  
  from order_info 
  where  (DATE_FORMAT(create_time,‘%Y-%m-%d‘)=‘$db_date‘ or DATE_FORMAT(operate_time,‘%Y-%m-%d‘)=‘$db_date‘)"
}

case $1 in
  "base_category1")
     import_base_category1
;;
  "base_category2")
     import_base_category2
;;
  "base_category3")
     import_base_category3
;;
  "order_info")
     import_order_info
;;
  "order_detail")
     import_order_detail
;;
  "sku_info")
     import_sku_info
;;
  "user_info")
     import_user_info
;;
  "payment_info")
     import_payment_info
;;
   "all")
   import_base_category1
   import_base_category2
   import_base_category3
   import_order_info
   import_order_detail
   import_sku_info
   import_user_info
   import_payment_info
;;
esac

hive (gmall)>
drop table if exists ods_order_info;
create external table ods_order_info ( 
    `id` string COMMENT ‘订单编号‘,
    `total_amount` decimal(10,2) COMMENT ‘订单金额‘, 
    `order_status` string COMMENT ‘订单状态‘, 
    `user_id` string COMMENT ‘用户id‘ ,
    `payment_way` string COMMENT ‘支付方式‘,  
    `out_trade_no` string COMMENT ‘支付流水号‘,  
    `create_time` string COMMENT ‘创建时间‘,  
    `operate_time` string COMMENT ‘操作时间‘ 
) COMMENT ‘订单表‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_order_info/‘
;

hive (gmall)>
drop table if exists ods_order_detail;
create external table ods_order_detail( 
    `id` string COMMENT ‘订单编号‘,
    `order_id` string  COMMENT ‘订单号‘, 
    `user_id` string COMMENT ‘用户id‘ ,
    `sku_id` string COMMENT ‘商品id‘,  
    `sku_name` string COMMENT ‘商品名称‘,  
    `order_price` string COMMENT ‘商品价格‘,  
    `sku_num` string COMMENT ‘商品数量‘,  
    `create_time` string COMMENT ‘创建时间‘
) COMMENT ‘订单明细表‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_order_detail/‘
;

hive (gmall)>
drop table if exists ods_sku_info;
create external table ods_sku_info( 
    `id` string COMMENT ‘skuId‘,
    `spu_id` string   COMMENT ‘spuid‘, 
    `price` decimal(10,2) COMMENT ‘价格‘ ,
    `sku_name` string COMMENT ‘商品名称‘,  
    `sku_desc` string COMMENT ‘商品描述‘,  
    `weight` string COMMENT ‘重量‘,  
    `tm_id` string COMMENT ‘品牌id‘,  
    `category3_id` string COMMENT ‘品类id‘,  
    `create_time` string COMMENT ‘创建时间‘
) COMMENT ‘商品表‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_sku_info/‘
;

hive (gmall)>
drop table if exists ods_user_info;
create external table ods_user_info( 
    `id` string COMMENT ‘用户id‘,
    `name`  string COMMENT ‘姓名‘, 
    `birthday` string COMMENT ‘生日‘ ,
    `gender` string COMMENT ‘性别‘,  
    `email` string COMMENT ‘邮箱‘,  
    `user_level` string COMMENT ‘用户等级‘,  
    `create_time` string COMMENT ‘创建时间‘
) COMMENT ‘用户信息‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_user_info/‘
;

hive (gmall)>
drop table if exists ods_base_category1;
create external table ods_base_category1( 
    `id` string COMMENT ‘id‘,
    `name`  string COMMENT ‘名称‘
) COMMENT ‘商品一级分类‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_base_category1/‘
;

hive (gmall)>
drop table if exists ods_base_category2;
create external table ods_base_category2( 
    `id` string COMMENT ‘ id‘,
    `name`  string COMMENT ‘名称‘,
    category1_id string COMMENT ‘一级品类id‘
) COMMENT ‘商品二级分类‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_base_category2/‘
;

hive (gmall)>
drop table if exists ods_base_category3;
create external table ods_base_category3( 
    `id` string COMMENT ‘ id‘,
    `name`  string COMMENT ‘名称‘,
    category2_id string COMMENT ‘二级品类id‘
) COMMENT ‘商品三级分类‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_base_category3/‘
;

hive (gmall)>
drop table if exists `ods_payment_info`;
create external table  `ods_payment_info`(
    `id`   bigint COMMENT ‘编号‘,
    `out_trade_no`    string COMMENT ‘对外业务编号‘,
    `order_id`        string COMMENT ‘订单编号‘,
    `user_id`         string COMMENT ‘用户编号‘,
    `alipay_trade_no` string COMMENT ‘支付宝交易流水编号‘,
    `total_amount`    decimal(16,2) COMMENT ‘支付金额‘,
    `subject`         string COMMENT ‘交易内容‘,
    `payment_type` string COMMENT ‘支付类型‘,
    `payment_time`   string COMMENT ‘支付时间‘
   )  COMMENT ‘支付流水表‘
PARTITIONED BY ( `dt` string)
row format delimited  fields terminated by ‘\t‘ 
location ‘/warehouse/gmall/ods/ods_payment_info/‘
;

[kris@hadoop101 bin]$ cat ods_db.sh 
#!/bin/bash
   APP=gmall
   hive=/opt/module/hive/bin/hive
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
        do_date=$1
else 
        do_date=`date -d "-1 day" +%F`  
fi
sql=" 
load data inpath ‘/origin_data/$APP/db/order_info/$do_date‘  OVERWRITE into table "$APP".ods_order_info partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/order_detail/$do_date‘  OVERWRITE into table "$APP".ods_order_detail partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/sku_info/$do_date‘  OVERWRITE into table "$APP".ods_sku_info partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/user_info/$do_date‘ OVERWRITE into table "$APP".ods_user_info partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/payment_info/$do_date‘ OVERWRITE into table "$APP".ods_payment_info partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/base_category1/$do_date‘ OVERWRITE into table "$APP".ods_base_category1 partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/base_category2/$do_date‘ OVERWRITE into table "$APP".ods_base_category2 partition(dt=‘$do_date‘);

load data inpath ‘/origin_data/$APP/db/base_category3/$do_date‘ OVERWRITE into table "$APP".ods_base_category3 partition(dt=‘$do_date‘); 
"
$hive -e "$sql"

hive (gmall)>
drop table if exists dwd_order_info;
create external table dwd_order_info ( 
    `id` string COMMENT ‘‘,
    `total_amount` decimal(10,2) COMMENT ‘‘, 
    `order_status` string COMMENT ‘ 1  2  3  4  5‘, 
    `user_id` string COMMENT ‘id‘ ,
    `payment_way` string COMMENT ‘‘,  
    `out_trade_no` string COMMENT ‘‘,  
    `create_time` string COMMENT ‘‘,  
    `operate_time` string COMMENT ‘‘ 
) COMMENT ‘‘
PARTITIONED BY ( `dt` string)
stored as  parquet
location ‘/warehouse/gmall/dwd/dwd_order_info/‘
tblproperties ("parquet.compression"="snappy")
;

hive (gmall)>
drop table if exists dwd_order_detail;
create external table dwd_order_detail( 
    `id` string COMMENT ‘‘,
    `order_id` decimal(10,2) COMMENT ‘‘, 
    `user_id` string COMMENT ‘id‘ ,
    `sku_id` string COMMENT ‘id‘,  
    `sku_name` string COMMENT ‘‘,  
    `order_price` string COMMENT ‘‘,  
    `sku_num` string COMMENT ‘‘, 
    `create_time` string COMMENT ‘‘
) COMMENT ‘‘
PARTITIONED BY ( `dt` string)
stored as  parquet
location ‘/warehouse/gmall/dwd/dwd_order_detail/‘
tblproperties ("parquet.compression"="snappy")
;

hive (gmall)>
drop table if exists dwd_user_info;
create external table dwd_user_info( 
    `id` string COMMENT ‘id‘,
    `name`  string COMMENT ‘‘, 
    `birthday` string COMMENT ‘‘ ,
    `gender` string COMMENT ‘‘,  
    `email` string COMMENT ‘‘,  
    `user_level` string COMMENT ‘‘,  
    `create_time` string COMMENT ‘‘
) COMMENT ‘‘
PARTITIONED BY ( `dt` string)
stored as  parquet
location ‘/warehouse/gmall/dwd/dwd_user_info/‘
tblproperties ("parquet.compression"="snappy")
;

hive (gmall)>
drop table if exists `dwd_payment_info`;
create external  table  `dwd_payment_info`(
    `id`   bigint COMMENT ‘‘,
    `out_trade_no`   string COMMENT ‘‘,
    `order_id`        string COMMENT ‘‘,
    `user_id`         string COMMENT ‘‘,
    `alipay_trade_no` string COMMENT ‘‘,
    `total_amount`    decimal(16,2) COMMENT ‘‘,
    `subject`         string COMMENT ‘‘,
    `payment_type` string COMMENT ‘‘,
    `payment_time`   string COMMENT ‘‘
   )  COMMENT ‘‘
PARTITIONED BY ( `dt` string)
stored as  parquet
location ‘/warehouse/gmall/dwd/dwd_payment_info/‘
tblproperties ("parquet.compression"="snappy")
;

hive (gmall)>
drop table if exists dwd_sku_info;
create external table dwd_sku_info( 
    `id` string COMMENT ‘skuId‘,
    `spu_id` string COMMENT ‘spuid‘, 
    `price` decimal(10,2) COMMENT ‘‘ ,
    `sku_name` string COMMENT ‘‘,  
    `sku_desc` string COMMENT ‘‘,  
    `weight` string COMMENT ‘‘,  
    `tm_id` string COMMENT ‘id‘,  
    `category3_id` string COMMENT ‘1id‘,  
    `category2_id` string COMMENT ‘2id‘,  
    `category1_id` string COMMENT ‘3id‘,  
    `category3_name` string COMMENT ‘3‘,  
    `category2_name` string COMMENT ‘2‘,  
    `category1_name` string COMMENT ‘1‘,  
    `create_time` string COMMENT ‘‘
) COMMENT ‘‘
PARTITIONED BY ( `dt` string)
stored as  parquet
location ‘/warehouse/gmall/dwd/dwd_sku_info/‘
tblproperties ("parquet.compression"="snappy")
;

[kris@hadoop101 bin]$ cat dwd_db.sh 
#!/bin/bash
APP=gmall
hive=/opt/module/hive/bin/hive

if [ -n "$1" ]; then
        do_date=$1
else
        do_date=`date -d ‘-1 day‘ +%F`
fi
sql="
set hive.exec.dynamic.partition.mode=nonstrict;
insert overwrite table   "$APP".dwd_order_info partition(dt)
select * from "$APP".ods_order_info 
where dt=‘$do_date‘  and id is not null;
 
insert overwrite table   "$APP".dwd_order_detail partition(dt)
select * from "$APP".ods_order_detail 
where dt=‘$do_date‘   and id is not null;

insert overwrite table   "$APP".dwd_user_info partition(dt)
select * from "$APP".ods_user_info
where dt=‘$do_date‘   and id is not null;
 
insert overwrite table   "$APP".dwd_payment_info partition(dt)
select * from "$APP".ods_payment_info
where dt=‘$do_date‘  and id is not null;

insert overwrite table   "$APP".dwd_sku_info partition(dt)
select  
    sku.id,
    sku.spu_id, 
    sku.price,
    sku.sku_name,  
    sku.sku_desc,  
    sku.weight,  
    sku.tm_id,  
    sku.category3_id,  
    c2.id category2_id ,  
    c1.id category1_id,  
    c3.name category3_name,  
    c2.name category2_name,  
    c1.name category1_name,  
    sku.create_time,
    sku.dt
from
    "$APP".ods_sku_info sku 
join "$APP".ods_base_category3 c3 on sku.category3_id=c3.id 
    join "$APP".ods_base_category2 c2 on c3.category2_id=c2.id 
    join "$APP".ods_base_category1 c1 on c2.category1_id=c1.id 
where sku.dt=‘$do_date‘  and c2.dt=‘$do_date‘  
and  c3.dt=‘$do_date‘ and  c1.dt=‘$do_date‘ 
and sku.id is not null;
"
$hive -e "$sql"

hive (gmall)>
drop table if exists dws_user_action;
create  external table dws_user_action 
(   
    user_id         string      comment ‘用户 id‘,
    order_count     bigint      comment ‘下单次数 ‘,
    order_amount    decimal(16,2)  comment ‘下单金额 ‘,
    payment_count   bigint      comment ‘支付次数‘,
    payment_amount  decimal(16,2) comment ‘支付金额 ‘,
    comment_count   bigint      comment ‘评论次数‘ 
) COMMENT ‘每日用户行为宽表‘
PARTITIONED BY ( `dt` string)
stored as  parquet 
location ‘/warehouse/gmall/dws/dws_user_action/‘
tblproperties ("parquet.compression"="snappy");

hive (gmall)>
with  
tmp_order as
(
    select 
        user_id, 
count(*)  order_count,
        sum(oi.total_amount) order_amount
    from dwd_order_info  oi
    where date_format(oi.create_time,‘yyyy-MM-dd‘)=‘2019-02-10‘
    group by user_id
)  ,
tmp_payment as
(
    select 
        user_id, 
        sum(pi.total_amount) payment_amount, 
        count(*) payment_count 
    from dwd_payment_info pi 
    where date_format(pi.payment_time,‘yyyy-MM-dd‘)=‘2019-02-10‘
    group by user_id
),
tmp_comment as
(  
    select  
        user_id, 
        count(*) comment_count
    from dwd_comment_log c
    where date_format(c.dt,‘yyyy-MM-dd‘)=‘2019-02-10‘
    group by user_id 
)

insert overwrite table dws_user_action partition(dt=‘2019-02-10‘)
select 
    user_actions.user_id, 
    sum(user_actions.order_count), 
    sum(user_actions.order_amount),
    sum(user_actions.payment_count), 
    sum(user_actions.payment_amount),
    sum(user_actions.comment_count) 
from 
(
    select 
        user_id, 
        order_count,
        order_amount ,
        0 payment_count , 
        0 payment_amount, 
        0 comment_count 
    from tmp_order 

    union all
    select 
        user_id, 
        0,
        0, 
        payment_count, 
        payment_amount,
        0  
    from tmp_payment

    union all
    select 
        user_id, 
        0,
        0,
        0,
        0,
        comment_count 
    from tmp_comment
 ) user_actions
group by user_id;

#!/bin/bash

# 定义变量方便修改
APP=gmall
hive=/opt/module/hive/bin/hive

# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
    do_date=$1
else 
    do_date=`date  -d "-1 day"  +%F`  
fi 

sql="

with  
tmp_order as
(
    select 
      user_id, 
      sum(oi.total_amount) order_amount, 
      count(*)  order_count
    from "$APP".dwd_order_info  oi
    where date_format(oi.create_time,‘yyyy-MM-dd‘)=‘$do_date‘
    group by user_id
)  ,
tmp_payment as
(
    select 
      user_id, 
      sum(pi.total_amount) payment_amount, 
      count(*) payment_count 
    from "$APP".dwd_payment_info pi 
    where date_format(pi.payment_time,‘yyyy-MM-dd‘)=‘$do_date‘
    group by user_id
),
tmp_comment as
(  
    select  
      user_id, 
      count(*) comment_count
    from "$APP".dwd_comment_log c
    where date_format(c.dt,‘yyyy-MM-dd‘)=‘$do_date‘
    group by user_id 
)

insert  overwrite table "$APP".dws_user_action partition(dt=‘$do_date‘)
select 
    user_actions.user_id, 
    sum(user_actions.order_count), 
    sum(user_actions.order_amount),
    sum(user_actions.payment_count), 
    sum(user_actions.payment_amount),
    sum(user_actions.comment_count) 
from 
(
    select 
      user_id, 
      order_count,
      order_amount ,
      0 payment_count , 
      0 payment_amount, 
      0 comment_count 
    from tmp_order 

    union all
    select 
      user_id, 
      0,
      0, 
      payment_count, 
      payment_amount,
      0  
    from tmp_payment

    union all
    select 
      user_id, 
      0,
      0,
      0,
      0,
      comment_count 
    from tmp_comment
 ) user_actions
group by user_id;

"

$hive -e "$sql"

-server
-Xmx16G
-XX:+UseG1GC
-XX:G1HeapRegionSize=32M
-XX:+UseGCOverheadLimit
-XX:+ExplicitGCInvokesConcurrent
-XX:+HeapDumpOnOutOfMemoryError
-XX:+ExitOnOutOfMemoryError