{"id":514,"date":"2020-04-30T00:48:36","date_gmt":"2020-04-29T16:48:36","guid":{"rendered":"https:\/\/qkongtao.cn\/?p=514"},"modified":"2023-03-16T16:51:22","modified_gmt":"2023-03-16T08:51:22","slug":"%e5%a4%a7%e4%bd%9c%e4%b8%9a-%e6%96%b0%e5%86%a0%e8%82%ba%e7%82%8e%e7%96%ab%e6%83%85%e7%9a%84%e6%95%b0%e6%8d%ae%e9%87%87%e9%9b%86%e5%92%8c%e5%8f%af%e8%a7%86%e5%8c%96%e5%88%86%e6%9e%90","status":"publish","type":"post","link":"https:\/\/qkongtao.cn\/?p=514","title":{"rendered":"\u5927\u4f5c\u4e1a\u2014\u2014\u65b0\u51a0\u80ba\u708e\u75ab\u60c5\u7684\u6570\u636e\u91c7\u96c6\u548c\u53ef\u89c6\u5316\u5206\u6790\u7cfb\u7edf"},"content":{"rendered":"

\u6982\u8ff0<\/h2>\n

\u8fd9\u6b21\u5927\u4f5c\u4e1a\u7684\u4e3b\u8981\u6d41\u7a0b\u662f\uff1a<\/strong>
\n\u9996\u5148\u8981\u91c7\u96c6\u6570\u636e\uff0c\u91c7\u7528\u811a\u672c\u5b9a\u65f6\u91c7\u96c6\u7684\u90a3\u79cd\uff0c\u91c7\u96c6\u7684\u6570\u636e\u6765\u6e90\u8fd9\u7bc7\u535a\u6587:https:\/\/www.dzyong.com\/#\/ViewArticle\/123\uff0c\u91cc\u9762\u6709\u51e0\u4e2a\u6570\u636e\u63a5\u53e3\uff0c\u8fd4\u56de\u7684\u6570\u636e\u662fjson\u683c\u5f0f\uff0c\u7528java\u7a0b\u5e8f\uff0c\u5148\u8f6c\u5316\u4e3a\u7528tab\u952e\u5206\u5272\u7684\u6587\u672c\u6570\u636e\uff0c\u7136\u540e\u5bfc\u5165hive\u4e2d\uff1b
\n\u5176\u6b21\u662f\u5728hive\u4e2d\u5bf9\u5bfc\u8fdb\u6765\u7684\u6570\u636e\u8fdb\u884c\u5904\u7406\u8fc7\u6ee4\uff0c\u518d\u5efa\u51e0\u4e2a\u8868\uff0c\u628a\u5904\u7406\u7ed3\u679c\u5b58\u5230\u65b0\u5efa\u7684\u8868\u91cc\uff0c\u7136\u540e\u628ahive\u5904\u7406\u7ed3\u679c\u7684\u6570\u636e\u8868\u5bfc\u5165mysql\u4e2d\uff1b\u8fd9\u6837\u505a\u5b8c\u4e00\u6b21\u540e\uff0c\u5f00\u59cb\u5199\u811a\u672c\uff0c\u6bcf\u9694\u4e00\u5929\u91c7\u96c6\u4e00\u6b21\u6570\u636e\uff0chive\u5904\u7406\u6570\u636e\u4e00\u6b21\uff0cmysql\u7edf\u8ba1\u6570\u636e\u4e00\u6b21\uff1b
\n\u63a5\u7740\u5c31\u662f\u7f16\u7a0b\uff0c\u7528ssm\u6846\u67b6\u8fde\u63a5\u5230mysql\uff0c\u5bf9\u6570\u636e\u7528javaBean\u8fdb\u884c\u5c01\u88c5\uff0c\u7528mvc\u6a21\u5f0f\u5c06\u90e8\u5206\u6570\u636e\u663e\u793a\u5230\u524d\u53f0\u9875\u9762\uff1b
\n\u6700\u540e\u7528echarts\u5bf9\u5c01\u88c5\u7684\u6570\u636e\u8fdb\u884c\u6570\u636e\u53ef\u89c6\u5316\uff0c\u53ef\u4ee5\u505a\u6210\u6761\u5f62\u56fe\uff0c\u6298\u7ebf\u56fe\uff0c\u997c\u56fe\uff0c\u6c14\u6ce1\u56fe\uff0c\u5730\u56fe\u7b49\u53ef\u89c6\u5316\u56fe\u6807\u3002<\/p>\n

\u6570\u636e\u6e05\u7406\u6d41\u7a0b\uff1a<\/strong>
\n1. \u9996\u5148\u6267\u884cGetData.jar\u5199\u597d\u7684\u7a0b\u5e8f\u83b7\u53d6\u6570\u636e\uff0c\u4f1a\u81ea\u52a8\u751f\u6210txt\u6570\u636e\u6587\u4ef6\u5728\/home\/kt\/devHive\/data\u6587\u4ef6\u5939\u91cc\u9762
\n2. \u7136\u540e\u6267\u884c\u5bfc\u5165\u6570\u636e\u5230\u5efa\u597d\u7684hive\u8868\u91cc\u9762\u7684\u811a\u672c
\n3. \u63a5\u7740\u6267\u884csql\uff0csql\u4f1a\u6267\u884cclean.sql\u91cc\u9762\u7684\u52a0\u5de5\u6570\u636e\u7684hql\u8bed\u53e5\uff0c\u4f1a\u5c06\u6e05\u7406\u597d\u7684\u6570\u636e\u5bfc\u5165Ed\u7684\u6e05\u6d01hive\u8868\u4e2d
\n4. \u63a5\u7740\u6267\u884chiveToMySql.sh\uff0c\u5c06<\/em>Ed\u8868\u91cc\u9762\u7684\u6e05\u6d01\u6570\u636e\u7528sqoop\u5bfc\u5165\u5bf9\u5e94\u7684MySQL\u8868\u4e2d\uff08\u4f1a\u6e05\u7a7a *Ed \u7684\u6240\u6709\u6570\u636e\uff09
\n5. \u6700\u540e\u53ef\u4ee5\u7528\u8fdc\u7a0b\u8fde\u63a5\u83b7\u53d6MySQL\u91cc\u7684\u6570\u636e<\/p>\n

\u6570\u636e\u91c7\u96c6\u6d41\u7a0b<\/h2>\n

1. \u7f16\u5199java\u7a0b\u5e8f\u91c7\u96c6\u6570\u636e<\/h3>\n

1.1 GetHttpInterface.java \u8c03\u7528http\u63a5\u53e3\u83b7\u53d6\u6570\u636e<\/h4>\n
package cn.qkongtao.getDataInterface;\/*\n *Created by tao on 2020-04-23.\n *\/\n\n\nimport com.alibaba.fastjson.JSONObject;\n\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.net.HttpURLConnection;\nimport java.net.URL;\n\npublic class GetHttpInterface {\n    \/\/ \u8c03\u7528http\u63a5\u53e3\u83b7\u53d6\u6570\u636e\n    public static String GetHttpInterface(String path) {\n        BufferedReader in = null;\n        StringBuffer result = null;\n        try {\n            URL url = new URL(path);\n            \/\/\u6253\u5f00\u548curl\u4e4b\u95f4\u7684\u8fde\u63a5\n            HttpURLConnection connection = (HttpURLConnection) url.openConnection();\n            connection.setRequestProperty("Content-Type", "application\/x-www-form-urlencoded");\n            connection.setRequestProperty("Charset", "utf-8");\n            connection.connect();\n\n            result = new StringBuffer();\n            \/\/\u8bfb\u53d6URL\u7684\u54cd\u5e94\n            in = new BufferedReader(new InputStreamReader(\n                    connection.getInputStream()));\n            String line;\n            while ((line = in.readLine()) != null) {\n                result.append(line);\n            }\n            \/\/\u89e3\u6790json\u5bf9\u8c61\n            JSONObject jsStr = (JSONObject) JSONObject.parse(result.toString());\n            return jsStr.get("data").toString();\n        } catch (Exception e) {\n            e.printStackTrace();\n        } finally {\n            try {\n                if (in != null) {\n                    in.close();\n                }\n            } catch (Exception e2) {\n                e2.printStackTrace();\n            }\n        }\n        return null;\n    }\n}<\/code><\/pre>\n
1.2 GetDataProvince.java  \u8c03\u7528\u63a5\u53e3\uff0c\u83b7\u5f97\u6570\u636e\u5199\u5165\u6587\u4ef6<\/h4>\n
package cn.qkongtao.getData;\/*\n *Created by tao on 2020-04-23.\n *\/\n\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\n\nimport static cn.qkongtao.getDataInterface.GetHttpInterface.GetHttpInterface;\n\npublic class GetDataProvince {\n    public void GetDataProvince(String path) {\n        String str = GetHttpInterface("http:\/\/www.dzyong.top:3005\/yiqing\/province");\n        str = str.replace("[", "");\n        str = str.replace("]", "");\n        String[] split = str.split("},");\n\/\/        System.out.println(str);\n        \/\/\u5c06\u6570\u636e\u5bfc\u5165\u6587\u672c\n        File file = new File(path);\n        if (file.exists()) {\n            System.out.println("\u6587\u4ef6\u5b58\u5728");\n        } else {\n            try {\n                file.createNewFile();\n            } catch (IOException e) {\n                e.printStackTrace();\n            }\n            System.out.println("\u65b0\u5efa\u6587\u4ef6\u6210\u529f");\n        }\n        FileOutputStream outputStream;\n        try {\n            outputStream = new FileOutputStream(file);\n            for (int i = 0; i <split.length ; i++) {\n\n\/\/                System.out.println(split[i]+"}\\r\\n");\n                outputStream.write((split[i]+"}\\r\\n").getBytes());\n            }\n\/\/            outputStream.write(str.getBytes());\n            outputStream.close();\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n}<\/code><\/pre>\n
1.3 getMain.java \u6d4b\u8bd5\u7c7b<\/h4>\n
package cn.qkongtao.getMain;\/*\n *Created by tao on 2020-04-24.\n *\/\n\nimport cn.qkongtao.getData.*;\n\npublic class getMain {\n    public static void main(String[] args) {\n        new GetDataArea().GetDataArea("\/home\/kt\/devHive\/data\/area.txt");\n        new GetDataHistory().GetDataHistory("\/home\/kt\/devHive\/data\/history.txt");\n        new GetDataNews().GetDataNews("\/home\/kt\/devHive\/data\/news.txt");\n        new GetDataProvince().GetDataProvince("\/home\/kt\/devHive\/data\/province.txt");\n        new GetDataTotal().GetDataTotal("\/home\/kt\/devHive\/data\/total.txt");\n    }\n}<\/code><\/pre>\n
1.4 \u6d4b\u8bd5\u7ed3\u679c<\/h4>\n
\"\u5927\u4f5c\u4e1a\u2014\u2014\u65b0\u51a0\u80ba\u708e\u75ab\u60c5\u7684\u6570\u636e\u91c7\u96c6\u548c\u53ef\u89c6\u5316\u5206\u6790\u7cfb\u7edf-Nick\u306e\u535a\u5ba2\"
\n\u6e90\u4ee3\u7801\u6587\u672b\u4f1a\u7ed9\u51fa<\/strong><\/p>\n
\u6570\u636e\u6e05\u6d17\u6d41\u7a0b<\/h2>\n
1. \u521b\u5efa\u76f8\u5173\u7684hive\u8868<\/h3>\n
\u6ce8\u610f\uff1a\u5bfc\u5165json\u6570\u636e\uff08\u9700\u8981\u5bfc\u5165\u5de5\u5177jar\u5305\uff1ajson-serde-1.3.7-jar-with-dependencies.jar\uff09<\/strong>
\nJsonSerde \u7684 github \u5730\u5740\uff1ahttps:\/\/github.com\/rcongiu\/Hive-JSON-Serde
\nJsonSerde \u7684 jar\u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/www.congiu.net\/hive-json-serde\/
\nJsonSerde \u7f51\u76d8\u4e0b\u8f7d\u5730\u5740\uff1ahttps:\/\/pan.baidu.com\/s\/1BexXxyR38Qas8OeunUQQGw
\n\u63d0\u53d6\u7801\uff1arv6l
\n\u6ce8\u610f\uff0c\u5728hive\u91cc\u9762date\u662f\u5173\u952e\u5b57\uff0c\u4e0d\u80fd\u76f4\u63a5\u4f5c\u4e3a\u5b57\u6bb5\uff0c\u5982\u679c\u4e00\u5b9a\u8981\u4f5c\u4e3a\u5b57\u6bb5\uff0c\u8981\u5728hive\u91cc\u9762\u6267\u884c\u4e0b\u9762\u7684\u8bed\u53e5<\/strong>
\nset hive.support.sql11.reserved.keywords=false;<\/p>\n
1.1 \u5386\u53f2\u6570\u636e\u8868<\/h4>\n
date \u5386\u53f2\u65e5\u671f,
\nconfirmedNum  \u5386\u53f2\u786e\u8bca\u75c5\u4f8b,
\nsuspectedNum  \u5386\u53f2\u7591\u4f3c\u75c5\u4f8b,
\ncuresNum  \u5386\u53f2\u6cbb\u6108\u75c5\u4f8b,
\ndeathsNum  \u5386\u53f2\u6b7b\u4ea1\u75c5\u4f8b,
\nsuspectedIncr  \u5386\u53f2\u7591\u4f3c\u75c5\u4f8b,<\/p>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE EXTERNAL TABLE history(\n   date string,\n   suspectedNum  int,\n   curesNum  int,\n   confirmedNum  int,\n   deathsNum  int,\n   id int\n   )ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'\nSTORED AS TEXTFILE;\n\n#\u5bfc\u5165\u6570\u636e\nload data local inpath '\/home\/kt\/devHive\/data\/history.txt' overwrite into table history;<\/code><\/pre>\n
select * from history;
\ndrop table history;<\/p>\n
1.2 \u83b7\u53d6\u5168\u56fd\u603b\u6570\u636e\u8868<\/h4>\n
\"diagnosed\": 24401,   \/\/\u786e\u8bca
\n\"suspect\": 23260,   \/\/\u7591\u4f3c
\n\"death\": 492,   \/\/\u6b7b\u4ea1
\n\"cured\": 967,   \/\/\u6cbb\u6108
\n\"date\": \"2020-02-05 15:36:16\"   \/\/\u66f4\u65b0\u65f6\u95f4<\/p>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE EXTERNAL TABLE total(\n   id int,\n   diagnosed  int,\n   suspect  int,\n   death  int,\n   cured  int,\n   date  string\n   )\nROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'\nSTORED AS TEXTFILE;\n\n#\u5bfc\u5165\u6570\u636e\nload data local inpath '\/home\/kt\/devHive\/data\/total.txt' overwrite into table total;<\/code><\/pre>\n
1.3 \u83b7\u53d6\u5404\u7701\/\u5e02\u6700\u65b0\u603b\u6570\u636e<\/h4>\n
provinceName  \u7701\u4efd\u540d\u79f0
\nconfirmedNum  \u5386\u53f2\u786e\u8bca\u75c5\u4f8b,
\ncuresNum  \u5386\u53f2\u6cbb\u6108\u75c5\u4f8b,
\ndeathsNum  \u5386\u53f2\u6b7b\u4ea1\u75c5\u4f8b,
\nsuspectedIncr  \u5386\u53f2\u7591\u4f3c\u75c5\u4f8b,<\/p>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE EXTERNAL TABLE province(\n   id int,\n   provinceName  string,\n   confirmedNum  int,\n   curesNum  int,\n   deathsNum  int\n   )\nROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'\nSTORED AS TEXTFILE;\n\n#\u5bfc\u5165\u6570\u636e\nload data local inpath '\/home\/kt\/devHive\/data\/province.txt' overwrite into table province;<\/code><\/pre>\n
1.4 \u83b7\u53d6\u5404\u7701\/\u5e02\/\u5730\u533a\u6570\u636e<\/h4>\n
\u540d\u79f0\u542b\u4e49\u53c2\u8003\u4e0a\u9762<\/p>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE EXTERNAL TABLE area(\n   id int,\n   provinceName  string,\n   cityName  string,\n   confirmedCount  int,\n   suspectedCount  int,\n   curedCount  int,\n   deadCount  int\n   )\nROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'\nSTORED AS TEXTFILE;\n\n#\u5bfc\u5165\u6570\u636e\nload data local inpath '\/home\/kt\/devHive\/data\/area.txt' overwrite into table area;<\/code><\/pre>\n
1.5 \u83b7\u53d6\u6700\u65b0\u52a8\u6001\u65b0\u95fb\uff08\u6700\u65e9\u52302\u67086\u65e5\u6570\u636e\uff09<\/h4>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE EXTERNAL TABLE news(\n   id int,\n   pubDate  string,\n   pubDateStr  string,\n   title  string,\n   summary  string,\n   infoSource  string,\n   sourceUrl  string,\n   provinceName  string,\n   createTime  string,\n   modifyTime  string\n   )\nROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'\nSTORED AS TEXTFILE;\n\n#\u5bfc\u5165\u6570\u636e\nload data local inpath '\/home\/kt\/devHive\/data\/news.txt' overwrite into table news;<\/code><\/pre>\n
2. hive\u5bf9\u6570\u636e\u5904\u7406\u7b5b\u9009\uff0c\u5bfc\u5165MySQL<\/h3>\n
2.1 \u5168\u56fd\u75ab\u60c5\u5206\u5e03\u5730\u56fe<\/h4>\n
\u786e\u8bca\u75c5\u4f8b
\n\u6b7b\u4ea1\u75c5\u4f8b
\n\u6cbb\u6108\u75c5\u4f8b<\/p>\n
create table provinceEd(\n   provinceName  string,\n   confirmedNum  int,\n   deathsNum  int,\n   curesNum  int\n)\nrow format delimited fields terminated by '\\t';<\/code><\/pre>\n
2.2 \u5404\u4e2a\u7701\u4efd\u7684\u75ab\u60c5\u5206\u5e03\u5730\u56fe<\/h4>\n
\u786e\u8bca\u75c5\u4f8b
\n\u6b7b\u4ea1\u75c5\u4f8b
\n\u6cbb\u6108\u75c5\u4f8b<\/p>\n
create table areaEd(\n   provinceName  string,\n   cityName  string,\n   confirmedCount  int,\n   deadCount  int,\n   curedCount  int\n)\nrow format delimited fields terminated by '\\t';<\/code><\/pre>\n
2.3 \u56fd\u5185\u75ab\u60c5\u8d8b\u52bf<\/h4>\n
\u786e\u8bca\u75c5\u4f8b
\n\u6b7b\u4ea1\u75c5\u4f8b
\n\u6cbb\u6108\u75c5\u4f8b<\/p>\n
create table historyEd(\n   date string,\n   confirmedNum  int,\n   deathsNum  int,\n   curesNum  int\n)\nrow format delimited fields terminated by '\\t';<\/code><\/pre>\n
2.4 \u5404\u7701\u5e02\u5730\u533a\u75ab\u60c5\u7684\u8868\u683c<\/h4>\n
\uff08\u7528historyEd,\u5e26\u6709\u5168\u56fd\u6570\u636e\u7684\u6700\u65b0\u6570\u636etotlaed\uff09<\/p>\n
#\u5efa\u8868\u8bed\u53e5\nCREATE TABLE totalEd(\n   date  string,\n   diagnosed  int,\n   death  int,\n   cured  int\n   )\nrow format delimited fields terminated by '\\t';<\/code><\/pre>\n
2.5 \u4e00\u4e2a\u65b0\u95fb\u7684\u4e13\u680f<\/h4>\n
pubData \u5177\u4f53\u65f6\u95f4
\ntitle \u65b0\u95fb\u6807\u9898
\nsummary \u65b0\u95fb\u8be6\u60c5
\ninfoSource \u65b0\u95fb\u6765\u6e90
\nsourceUrl \u65b0\u95fb\u6b63\u6587\u94fe\u63a5\u4f20\u9001<\/p>\n
CREATE  TABLE newsEd(\n   pubDate  string,\n   title  string,\n   summary  string,\n   infoSource  string,\n   sourceUrl  string,\n   provinceName  string\n   )\nrow format delimited fields terminated by '\\t';<\/code><\/pre>\n
3. \u5728MySQL\u4e2d\u521b\u5efa\u5bf9\u5e94\u7b5b\u9009\u540e5\u4e2a\u8868<\/h3>\n
\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/p>\n
4. \u7f16\u5199Linux\u811a\u672c<\/h3>\n
4.1 \u5bfc\u5165\u6570\u636e\u7684hql\u8bed\u53e5(load.sql)<\/h4>\n
set hive.exec.mode.local.auto=true;\nset hive.support.sql11.reserved.keywords=false;\nuse kongtao;\n\nload data local inpath '\/home\/kt\/devHive\/data\/history.txt' overwrite into table history;\n\nload data local inpath '\/home\/kt\/devHive\/data\/total.txt' overwrite into table total;\n\nload data local inpath '\/home\/kt\/devHive\/data\/province.txt' overwrite into table province;\n\nload data local inpath '\/home\/kt\/devHive\/data\/area.txt' overwrite into table area;\n\nload data local inpath '\/home\/kt\/devHive\/data\/news.txt' overwrite into table news;<\/code><\/pre>\n
4.2 \u83b7\u53d6\u6570\u636e\u7684\u811a\u672c getData.sh<\/h4>\n
#!\/bin\/bash\n. \/etc\/profile\nHIVE_HOME=\/app\/hive\/\nyesterday=`date -d -0days '+%Y%m%d'`\nhour=`date -d -0hour '+%H'`\necho $yesterday\n${HIVE_HOME}\/bin\/hive --hiveconf daily_param=${yesterday} \\\n--hiveconf hour_param=${hour} \\\n -f \/home\/kt\/devHive\/0425\/loa.sql\ndate >> \/var\/log\/httpd\/hivetToMysql.log\necho ${yesterday}${hour} >> \/home\/kt\/devHive\/log\/hivetToMysql.log<\/code><\/pre>\n
4.3 clean.sql\u8bed\u53e5\u811a\u672c<\/h4>\n
set hive.exec.mode.local.auto=true;\nset hive.support.sql11.reserved.keywords=false;\n\ninsert into table kongtao.provinceEd\nselect  provinceName,confirmedNum,deathsNum,curesNum from province;\n\ninsert into table kongtao.areaEd\nselect provinceName, cityName, confirmedCount,deadCount,curedCount from area;\n\ninsert into table kongtao.historyEd\nselect date, confirmedNum, deathsNum,curesNum from history;\n\ninsert into table kongtao.totalEd\nselect date, diagnosed, death,cured from total;\n\ninsert into table kongtao.newsEd\nselect pubDate, title, summary,infoSource,sourceUrl,provinceName from news;<\/code><\/pre>\n
4.4 \u5b9a\u65f6\u6267\u884cclean.sql\u7684\u8bed\u53e5 sql.sh<\/h4>\n
#!\/bin\/bash\n. \/etc\/profile\nHIVE_HOME=\/app\/hive\/\nyesterday=`date -d -0days '+%Y%m%d'`\nhour=`date -d -0hour '+%H'`\necho $yesterday\n${HIVE_HOME}\/bin\/hive --hiveconf daily_param=${yesterday} \\\n--hiveconf hour_param=${hour} \\\n -f \/home\/kt\/devHive\/0425\/clean.sql\ndate >> \/var\/log\/httpd\/hivetToMysql.log\necho ${yesterday}${hour} >> \/home\/kt\/devHive\/log\/hivetToMysql.log<\/code><\/pre>\n
hive -e \"use ${kongtao};select * from province;\"
\nchmod +x sql.sh<\/p>\n
4.5 \u5b9a\u65f6\u6267\u884chive\u5bfc\u5165MySQL\u7684\u8bed\u53e5<\/h4>\n
\u6ce8\u610f\uff1aMySQL\u5efa\u8868\u7684\u65f6\u5019\u8981\u8bbe\u7f6e\u5b57\u7b26\u7f16\u7801\uff0c\u5426\u5219\u4f1a\u5b57\u7b26\u4e0d\u5339\u914d\u5bfc\u4e0d\u8fdb\u53bb<\/strong>
\nENGINE=InnoDB DEFAULT CHARSET=utf8<\/p>\n
sqoop\u5bfc\u5165hive\u6570\u636e\u5230MySql\u78b0\u5230hive\u8868\u4e2d\u5217\u7684\u503c\u5982\u679c\u4e3anull\u7684\u60c5\u51b5\uff0chive\u4e2d\u4e3anull\u7684\u662f\u4ee5\\N\u4ee3\u66ff\u7684\uff0c\u6240\u4ee5\u4f60\u5728\u5bfc\u5165\u5230MySql\u65f6\uff0c\u9700\u8981\u52a0\u4e0a\u4e24\u4e2a\u53c2\u6570\uff1a--input-null-string '\\N' --input-null-non-string '\\N'\uff0c\u591a\u52a0\u4e00\u4e2a'\\'\uff0c\u662f\u4e3a\u8f6c\u4e49<\/p>\n
#!\/bin\/bash\n. \/etc\/profile\n\n#\u5148\u6e05\u695a\u8868\u4e2d\u7684\u6240\u6709\u6570\u636e\nhost="kt01"\nport="3306"\nuserName="root"\npassword="123456"\ndbname="kongtao"\ndbset="--default-character-set=utf8 -A"\n#\u5148\u6e05\u7a7a\u6240\u6709\u7684\u8868\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u91cd\u590d\ncmd="\ntruncate table areaEd;\ntruncate table historyEd;\ntruncate table totalEd;\ntruncate table provinceEd;\n"\n\nmysql -h${host} -u${userName} -p${password} ${dbname} -P${port}      -e  "${cmd}" \n\n#\u5bfc\u5165areaed\u8868\nsqoop export \\\n--connect "jdbc:mysql:\/\/kt01:3306\/kongtao?useUnicode=true&characterEncoding=utf-8" \\\n--username root \\\n--password 123456 \\\n--table areaEd \\\n--num-mappers 1 \\\n--input-fields-terminated-by "\\t" \\\n--export-dir \/user\/hive\/warehouse\/kongtao.db\/areaed\n#\u5bfc\u5165historyed\u8868\nsqoop export \\\n--connect "jdbc:mysql:\/\/kt01:3306\/kongtao?useUnicode=true&characterEncoding=utf-8" \\\n--username root \\\n--password 123456 \\\n--table historyEd \\\n--num-mappers 1 \\\n--input-fields-terminated-by "\\t" \\\n--export-dir \/user\/hive\/warehouse\/kongtao.db\/historyed\n#\u5bfc\u5165totaled\u8868\nsqoop export \\\n--connect "jdbc:mysql:\/\/kt01:3306\/kongtao?useUnicode=true&characterEncoding=utf-8" \\\n--username root \\\n--password 123456 \\\n--table totalEd \\\n--num-mappers 1 \\\n--input-fields-terminated-by "\\t" \\\n--export-dir \/user\/hive\/warehouse\/kongtao.db\/totaled\n#\u5bfc\u5165provinceed\u8868\nsqoop export \\\n--connect "jdbc:mysql:\/\/kt01:3306\/kongtao?useUnicode=true&characterEncoding=utf-8" \\\n--username root \\\n--password 123456 \\\n--table provinceEd \\\n--num-mappers 1 \\\n--input-fields-terminated-by "\\t" \\\n--export-dir \/user\/hive\/warehouse\/kongtao.db\/provinceed\n\n\/app\/hadoop\/bin\/hdfs dfs -rm -r \/user\/hive\/warehouse\/kongtao.db\/*ed\n\ndate >> \/home\/kt\/devHive\/log\/hivetToMysql.log<\/code><\/pre>\n
chmod +x hiveToMySql.sh<\/p>\n
5. \u7ed9\u811a\u672c\u8bbe\u7f6e\u5b9a\u65f6\u5668<\/h3>\n
crontab -e<\/p>\n
30 8 * * * \/home\/kt\/devHive\/0425\/getData.sh\n32 8 * * * \/home\/kt\/devHive\/0425\/sql.sh\n34 8 * * * \/home\/kt\/devHive\/0425\/hiveToMySql.sh<\/code><\/pre>\n
\u4f7f\u7528SSM\u7f16\u5199\u75ab\u60c5\u53ef\u89c6\u5316\u5206\u6790\u7cfb\u7edf<\/h2>\n
1. \u5b9e\u73b0\u7684\u529f\u80fd<\/h3>\n
    \n
  • \u5404\u7701\u5e02\u5730\u533a\u7684\u6570\u636e\u8868\u683c\uff08\u5e26\u6709\u5168\u56fd\u6570\u636e\u7684\u6700\u65b0\u6570\u636e\uff09<\/li>\n
  • \u5168\u56fd\u75ab\u60c5\u5206\u5e03\u5730\u56fe<\/li>\n
  • \u5404\u4e2a\u7701\u4efd\u7684\u75ab\u60c5\u5206\u5e03\u5730\u56fe<\/li>\n
  • \u56fd\u5185\u75ab\u60c5\u8d8b\u52bf<\/li>\n
  • \u6a21\u7cca\u67e5\u8be2\u7701\u4efd\u6216\u8005\u5177\u4f53\u57ce\u5e02<\/li>\n<\/ul>\n
    2. \u6548\u679c\u5982\u4e0b\uff1a<\/h3>\n
      \n
    1. \u9996\u9875
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n
    2. \u56fd\u5185\u75ab\u60c5\u8d8b\u52bf\u53ef\u89c6\u5316
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n
    3. \u5168\u56fd\u7701\u4efd\u75ab\u60c5\u6570\u636e\u8be6\u60c5
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n
    4. \u56fd\u5185\u75ab\u60c5\u5206\u5e03\u5730\u56fe
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n
    5. \u5404\u7701\u4efd\u7684\u6240\u6709\u5730\u533a\u6570\u636e\u8be6\u60c5
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n
    6. \u5404\u7701\u5206\u7684\u75ab\u60c5\u5206\u5e03\u5730\u56fe
      \n\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/li>\n<\/ol>\n
      \u603b\u7ed3\u2014\u2014\u6e90\u7801\u4e0b\u8f7d<\/h2>\n
         \u8fd9\u6b21\u505a\u7684\u53ef\u89c6\u5316\u5206\u6790\u7cfb\u7edf\u603b\u7684\u6765\u8bf4\u8fbe\u5230\u4e86\u81ea\u5df1\u671f\u5f85\u7684\u6548\u679c\uff0c\u7531\u4e8e\u53ea\u662f\u4e00\u4e2a\u5927\u4f5c\u4e1a\uff0c\u6240\u4ee5\u6ca1\u6709\u505a\u7684\u5f88\u7cbe\u7ec6\uff0c\u4f46\u60f3\u8981\u5b9e\u73b0\u7684\u529f\u80fd\u90fd\u5b9e\u73b0\u4e86\u3002\u7528\u5230\u7684\u6280\u672f\u4e3b\u8981\u6709\u5927\u6570\u636e\u90a3\u4e00\u5757\uff1ashell\u811a\u672c\uff0chive\uff0csqoop\uff0cMySQL\uff1bSSM\u6846\u67b6\u6574\u5408\uff1b\u524d\u7aef\u57fa\u672c\u6ca1\u6709\u5199\u4ec0\u4e48\u6837\u5f0f\uff0c\u7b80\u5355\u7684\u4f7f\u7528\u4e86bootstrap\uff1b\u548c\u6570\u636e\u53ef\u89c6\u5316\u5de5\u5177\uff1aecharts\u3002
      \n   \u521a\u5f00\u59cb\u60f3\u8981\u81ea\u5df1\u5199\u722c\u866b\u7a0b\u5e8f\u6765\u83b7\u53d6\u6570\u636e\uff0c\u4f46\u662f\u65f6\u95f4\u6bd4\u8f83\u7d27\uff0c\u5bf9\u722c\u866b\u4e5f\u4e0d\u719f\u6089\uff0c\u6240\u4ee5\u4f7f\u7528\u4e86\u73b0\u6210\u7684\u63a5\u53e3\u6765\u91c7\u96c6\u6570\u636e\uff0c\u867d\u7136\u6570\u636e\u91c7\u96c6\u7684\u8fc7\u7a0b\u6709\u70b9\u50cf\u6a21\u62df\u7684\u6570\u636e\uff0c\u4f46\u662f\u6574\u4e2a\u6d41\u7a0b\u8fd8\u662f\u6bd4\u8f83\u5b8c\u5584\u7684\uff0c\u7f3a\u7684\u53ea\u662f\u722c\u866b\u83b7\u53d6\u6570\u636e\u7684\u8fc7\u7a0b\u3002
      \n   \u4e00\u4e2a\u661f\u671f\u7684\u8bfe\u7a0b\u8bbe\u8ba1\u3002\u3002\u3002
      \n   \u7ee7\u7eed\u52a0\u6cb9\u5427 ^ _ ^<\/p>\n
      \u6e90\u7801\u4e0b\u8f7d<\/h3>\n
      \u94fe\u63a5\uff1ahttps:\/\/pan.baidu.com\/s\/1koDvSoxlEIAPPjmGZf4gAw
      \n\u63d0\u53d6\u7801\uff1akppw<\/p>\n","protected":false},"excerpt":{"rendered":"
      \u6982\u8ff0 \u8fd9\u6b21\u5927\u4f5c\u4e1a\u7684\u4e3b\u8981\u6d41\u7a0b\u662f\uff1a \u9996\u5148\u8981\u91c7\u96c6\u6570\u636e\uff0c\u91c7\u7528\u811a\u672c\u5b9a\u65f6\u91c7\u96c6\u7684\u90a3\u79cd\uff0c\u91c7\u96c6\u7684\u6570\u636e\u6765\u6e90\u8fd9\u7bc7\u535a\u6587:https:\/ […]<\/p>\n","protected":false},"author":1,"featured_media":765,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[108,63,124,43],"tags":[82,137,132],"class_list":["post-514","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ssm","category-63","category-bigdata","category-studynote","tag-java","tag-ssm","tag-132"],"_links":{"self":[{"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/posts\/514","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=514"}],"version-history":[{"count":0,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/posts\/514\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=\/wp\/v2\/media\/765"}],"wp:attachment":[{"href":"https:\/\/qkongtao.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=514"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=514"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/qkongtao.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=514"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}