第7单商品推荐案例 Ver1.2-20221118

采集到mysql

采集到Mongodb

转移到hive

转换到hbase

生成hive结果

生成hbase结果

转移到mysql

转移到mysql

CREATE TABLE `test`.`goods_data` (
`mid` int(11) NOT NULL AUTO_INCREMENT,
`collector` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
`coll_time` datetime DEFAULT NULL,
`title` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
`raw_title` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
`pic_url` varchar(145) COLLATE utf8_unicode_ci DEFAULT NULL,
`detail_url` varchar(145) COLLATE utf8_unicode_ci DEFAULT NULL,
`view_price` decimal(10,2) DEFAULT 0.0,
`view_fee` decimal(10,2) DEFAULT 0.0,
`item_loc` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
`view_sales` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
`comment_count` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
PRIMARY KEY (`mid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

CREATE TABLE `wc_goods_name` (
`mid` int(11) NOT NULL AUTO_INCREMENT,
`collector` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
`coll_time` datetime DEFAULT NULL,
`word` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL,
`acount` int(11) DEFAULT '0',
PRIMARY KEY (`mid`)
) ENGINE=InnoDB AUTO_INCREMENT=1767 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

以下命令中'张三'需更换为本人的中文姓名，需与采集数据时使用的名字一致

sqoop import --connect "jdbc:mysql://home.hddly.cn:53306/test?useSSL=false&useUnicode=true&characterEncoding=utf-8" --username test --password test --table goods_data --where "collector='张三'" --delete-target-dir --target-dir /user/myname/sqoop_hive_goods --hive-database myname --hive-import --hive-overwrite --hive-table goods_data --hive-drop-import-delims --m 1

use myname;
drop table wc_goods_name;
create table wc_goods_name as (select 0 as mid,'张三' as collecter,current_timestamp() as coll_time,word, count(1) as acount from (select explode(split(title, ' ')) as word from goods_data) w group by word order by acount desc);

sqoop export --connect "jdbc:mysql://home.hddly.cn:53306/test?useSSL=false&useUnicode=true&characterEncoding=utf-8" --username test --password test --table wc_goods_name --fields-terminated-by '\001' --lines-terminated-by '\n' --export-dir /user/hive/warehouse/myname.db/wc_goods_name;

打开jupyter

导入bigdata库

创建脚本

编写脚本

查看该目录下是否含有本人学号的图片文件