一丶使用的工具
MapReduce Hbase Echarts IDEA2019.3
二丶实现思路
1.对需要处理的爬取数据进行处理。(因为是课堂测试，已经给出数据，并且很有规律性)
2.使用mapreduce 根据相应的条件对数据进行统计。
3.通过用户选择的条件将读取的数据转换成JSON格式，便于Echarts 展示使用
三丶运行代码
清洗数据并插入HBASE 和 MYSQL (由于HBASE 数据库操作不是很熟悉，就也存储到Mysql 数据库了)
package Com.Hbase; import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; public class test4 { public static Configuration configuration; public static Connection connection; public static Admin admin; public static void main(String[] args) { configuration = HBaseConfiguration.create(); configuration.set("hbase.zookeeper.quorum","192.168.58.128:2181,192.168.58.129:2181,192.168.58.130:2181,192.168.58.131:2181"); try { connection = ConnectionFactory.createConnection(configuration); admin = connection.getAdmin(); } catch (IOException e) { e.printStackTrace(); } java.sql.Connection con; //jdbc驱动 String driver="com.mysql.cj.jdbc.Driver"; //这里我的数据库是cxxt String url="jdbc:mysql://localhost:3306/xuqiuzhengji?&useSSL=false&serverTimezone=UTC"; String user="root"; String password="123456"; try { //注册JDBC驱动程序 Class.forName(driver); //建立连接 con = DriverManager.getConnection(url, user, password); if (!con.isClosed()) { System.out.println("数据库连接成功"); } try { int a=1000; int i = 0; while(i<1217){ String rowkey=String.valueOf(a); String ip = getData("data", rowkey, "datadetail", "ip"); String date = getData("data", rowkey, "datadetail", "date"); String day = getData("data", rowkey, "datadetail", "day"); String traffic = getData("data", rowkey, "datadetail", "traffic"); String type = getData("data", rowkey, "datadetail", "type"); String id = getData("data", rowkey, "datadetail", "id"); Statement stmt = con.createStatement(); // 必须要写入id值，否则插入错误，建立数据库时，不是设置id值自动，为何还必须写入？？？ String sql = "insert into datas(ip,date,day,traffic,type,id) values ('"+ip+"', '"+date+"','"+day+"','"+traffic+"','"+type+"','"+id+"')"; // String sql = "insert into user(name,password,status,oname,postalcode,linkman,attribute,brief,idnum,xqzjqx,llxqqx,grxxqx,tjcxqx) values(?,?,?,?,?,?,?,?,?,?,?,?,?)"; stmt.executeUpdate(sql); System.out.println("ip"+ip); i++; a++; } } catch (IOException e) { e.printStackTrace(); } close(); con.close(); } catch (ClassNotFoundException e) { System.out.println("数据库驱动没有安装"); } catch (SQLException e) { e.printStackTrace(); System.out.println("数据库连接失败"); } } public static String getData(String tableName, String rowKey, String colFamily, String col) throws IOException { Table table = connection.getTable(TableName.valueOf(tableName)); Get get = new Get(rowKey.getBytes()); get.addColumn(colFamily.getBytes(), col.getBytes()); Result result = table.get(get); Cell[] cells = result.rawCells(); String sss= ""; for (Cell cell : cells) { System.out.println("RowName:" + new String(CellUtil.cloneRow(cell)) + " "); System.out.println("Timetamp:" + cell.getTimestamp() + " "); System.out.println("column Family:" + new String(CellUtil.cloneFamily(cell)) + " "); System.out.println("row Name:" + new String(CellUtil.cloneQualifier(cell)) + " "); System.out.println("value:" + new String(CellUtil.cloneValue(cell)) + " "); sss=new String(CellUtil.cloneValue(cell)); } //showCell(result); table.close(); return sss; } public static String showCell(Result result) { Cell[] cells = result.rawCells(); String sss= ""; for (Cell cell : cells) { System.out.println("RowName:" + new String(CellUtil.cloneRow(cell)) + " "); System.out.println("Timetamp:" + cell.getTimestamp() + " "); System.out.println("column Family:" + new String(CellUtil.cloneFamily(cell)) + " "); System.out.println("row Name:" + new String(CellUtil.cloneQualifier(cell)) + " "); System.out.println("value:" + new String(CellUtil.cloneValue(cell)) + " "); sss=new String(CellUtil.cloneValue(cell)); } return sss; } public static void close() { try { if (admin != null) { admin.close(); } if (null != connection) { connection.close(); } } catch (IOException e) { e.printStackTrace(); } } }
HbaseAPI 连接Hbase 数据库
package Com.Hbase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.util.Bytes; import org.checkerframework.checker.units.qual.C; import org.omg.CORBA.StringHolder; import java.io.IOException; /** * DDL: *1.判断表是否存在 * 2.创建命名空间 * 3.创建命名空间 * 4.删除表 * * DML: * 5.插入数据 * 6.查数据（get） * 7.查数据 (scan) * 8.删除数据 */ public class HbaseAPI { private static Connection connection =null; private static Admin admin=null; static { try { // 获取赋值信息 Configuration configuration=HBaseConfiguration.create(); configuration.set("hbase.zookeeper.quorum","hadoop102,hadoop103,hadoop104"); //创建连接对象 connection=ConnectionFactory.createConnection(configuration); //创建Admin 对象 admin =connection.getAdmin(); } catch (IOException e) { e.printStackTrace(); } } // 1 .判断表是否存在 public static boolean isTableExist(String tablename) throws IOException { //获取配置信息 ///HBaseConfiguration configuration =new HBaseConfiguration(); Configuration configuration= HBaseConfiguration.create(); configuration.set("hbase.zookeeper.quorum","hadoop102,hadoop103,hadoop104"); configuration.set("hbase.zookeeper.property.clientPort", "2181"); //获取管理员对象 Connection connection = ConnectionFactory.createConnection(configuration); Admin admin = connection.getAdmin(); //HBaseAdmin admin = new HBaseAdmin(configuration); //判断表是否存在 boolean exists = admin.tableExists(TableName.valueOf(tablename)); //返回结果 return exists; } // 2. 创建表 可变形参 public static void creatTable(String tablename,String... cfs) throws IOException { //判断是否存在列族信息 if(cfs.length <=0){ System.out.println("请设置列族信息！"); return; } //表存在 if (isTableExist(tablename)){ System.out.println(tablename+"已经存在！"); return; } //创建表描述器 HTableDescriptor hTableDescriptor= new HTableDescriptor(TableName.valueOf(tablename)); //循环添加列族信息 for (String cf:cfs){ //创建列族信息 HColumnDescriptor hColumnDescriptor= new HColumnDescriptor(cf); //添加具体列族信息 hTableDescriptor.addFamily(hColumnDescriptor); } //创建表 admin.createTable(hTableDescriptor); } // 3. 删除表 public static void dropTable(String tableName) throws IOException { //表是否存在 if(!isTableExist(tableName)){ System.out.println(tableName+"表不存在！"); return; } //让表下线 admin.disableTable(TableName.valueOf(tableName)); //删除表 admin.deleteTable(TableName.valueOf(tableName)); System.out.println("删除表成功。"); } // 4. 创建命名空间 public static void createNameSpace(String ns){ //创建命名空间描述器 NamespaceDescriptor namespaceDescriptor =NamespaceDescriptor.create(ns).build(); //创建命名空间 try { admin.createNamespace(namespaceDescriptor); System.out.println("命名空间创建成功。"); }catch (NamespaceNotFoundException e){ System.out.println(ns+"命名空间已存在！"); } catch (IOException e) { e.printStackTrace(); } // System.out.println("虽然存在，仍旧到这里了"); } //5. 向表插入数据 public static void putData(String tableName,String rowKey,String cf,String cn,String value) throws IOException { // 获取表对象 Table table = connection.getTable(TableName.valueOf(tableName)); // 创建put 对象 Put put = new Put(Bytes.toBytes(rowKey)); //给put 对象赋值 put.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn),Bytes.toBytes(value)); //批量添加 // put.addColumn(Bytes.toBytes(cf),Bytes.toBytes("sex"),Bytes.toBytes("male")); //插入数据 table.put(put); //关闭连接 table.close(); } //6.获取数据 (get) public static void getData(String tableName, String rowKey, String cf,String cn) throws IOException { //获取表对象 Table table= connection.getTable(TableName.valueOf(tableName)); //创建Get对象 Get get = new Get(Bytes.toBytes(rowKey)); //2.1 获取指定列族 //get.addFamily(Bytes.toBytes(cf)); //2.2 获取指定列和族 // get.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn)); //2.3 获取数据的版本数 // get.getMaxVersions(); //获取数据 Result result = table.get(get); //解析result 并打印 for (Cell cell : result.rawCells()){ //打印数据 System.out.println("CF:"+Bytes.toString(CellUtil.cloneFamily(cell))+ ",CN:"+ Bytes.toString(CellUtil.cloneQualifier(cell))+ ",Value:"+Bytes.toString(CellUtil.cloneValue(cell))); } //关闭表连接 table.close(); } //7 获取数据 scan public static void scanTable(String tableName) throws IOException { //获取表对象 Table table= connection.getTable(TableName.valueOf(tableName)); //构建Scan 对象 // Scan scan = new Scan(); // 左闭右开 Scan scan = new Scan(Bytes.toBytes("1001"),Bytes.toBytes("1002")); //扫描表 ResultScanner scanner = table.getScanner(scan); //解析resultScanner for (Result result :scanner){ //解析 result 并打印 for(Cell cell : result.rawCells()){ //打印数据 System.out.println("rowkey:"+Bytes.toString(CellUtil.cloneRow(cell))+",CF:"+Bytes.toString(CellUtil.cloneFamily(cell))+ ",CN:"+ Bytes.toString(CellUtil.cloneQualifier(cell))+ ",Value:"+Bytes.toString(CellUtil.cloneValue(cell))); } } //关闭表连接 table.close(); } //8.删除数据 public static void deleteData(String tableName,String rowKey,String cf,String cn) throws IOException { //获取表对象 Table table = connection.getTable(TableName.valueOf(tableName)); //构建删除对象 Delete delete = new Delete(Bytes.toBytes(rowKey)); //2.1设置删除的列 // delete.addColumn(); // delete.addColumns(); //删除操作 table.delete(delete); //关闭连接 table.close(); } //关闭连接 public static void close() { if (admin!=null){ try { admin.close(); } catch (IOException e) { e.printStackTrace(); } } if (connection!=null){ try { connection.close(); } catch (IOException e) { e.printStackTrace(); } } } public static void main(String[] args) throws IOException { //测试表是否存在 System.out.println(isTableExist("shuju")); //创建表测试 // creatTable("XuQiuDemand","infos"); //System.out.println(isTableExist("XuQiuDemand")); //删除表操作 // dropTable("stu5"); //创建命名空间测试 // createNameSpace("cuixingyu"); //创建数据测试 // String sjz="test"; // putData("stu","1500","info2","names",sjz); //获取单行数据 // getData("stu","1001","info2","phone"); // System.out.println(isTableExist("stu5")); //scanTable("stu"); //测试删除 // deleteData("stu","1005","ss","ss");‘、+ close(); } }
图标联动
实现界面（需要引入js数据和jar 包）:
JSP 实现界面:
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> <%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> <!DOCTYPE html> <html> <head> <meta charset="UTF-8"> <!-- <script src='https://cdn.bootcss.com/echarts/3.7.0/echarts.simple.js'></script> --> <script src="js/jquery-1.8.3.min.js"></script> <script src="js/echarts.js"></script> <title>图表联动</title> </head> <body> <form action="getinfo" method="post"> 选择图表类型： <select id="tblx" name="tblx" class="form-control" style=" 150px;display: inline"> <option value='line'>折线图</option> <option value='bar'>柱形图</option> <option value='pie'>饼状图</option> </select> <br> 选择数据类型： <select id="type" name="type" class="form-control" style=" 150px;display: inline"> <option value='addr'>ip</option> <option value='cishu'>访问次数</option> <option value='type'>类型</option> </select> <br> <button type="submit">查询</button> </form> <!--为echarts准备一个容器，画图就画在里面--> <div id="box" style=" 1200px;height: 400px;margin-left: 4%"></div> <table style=" 1200px;margin-left: 10%" border="1" > <thead> <tr> <th>信息</th> <th>数目</th> </tr> </thead> <tbody id="body"> <c:forEach var="item" items="${info}"> <tr> <td>${item.info}</td> <td>${item.num}</td> </tr> </c:forEach> </tbody> </table> </body> <script type="text/javascript"> var type='${type}'; alert(type); var mydataX = new Array(0); var mydataY = new Array(0); var mydata = new Array(0); var data = '${json}'; var json = eval('(' + data + ')'); for(var i=0;i<json.length;i++){ mydataX.push(json[i].info); mydataY.push(json[i].num); var t={'name':json[i].info,'value':json[i].num}; mydata.push(t); } var myChart=echarts.init(document.getElementById("box")); //指定图表的配置项和数据 if(type=="pie"){ var option = { title: { text: '饼状数据图', subtext: '纯属虚构', left: 'center' }, tooltip: { trigger: 'item', formatter: '{a} <br/>{b} : {c} ({d}%)' }, legend: { orient: 'vertical', left: 'left', data: mydataX }, series: [ { name: '访问来源', type: 'pie', radius: '55%', center: ['50%', '60%'], data: mydata, emphasis: { itemStyle: { shadowBlur: 10, shadowOffsetX: 0, shadowColor: 'rgba(0, 0, 0, 0.5)' } } } ] }; } else if(type=="line"){ var option={ //标题 title:{ text:'数据展示 折线图' }, //工具箱 //保存图片 toolbox:{ show:true, feature:{ saveAsImage:{ show:true } } }, //图例-每一条数据的名字叫销量 legend:{ data:['数量'] }, //x轴 xAxis:{ data:mydataX }, //y轴没有显式设置，根据值自动生成y轴 yAxis:{}, //数据-data是最终要显示的数据 series:[{ name:'数量', type:'line', data:mydataY }] }; }else{ option = { xAxis: { type: 'category', data: mydataX }, yAxis: { type: 'value' }, series: [{ data: mydataY, type: 'bar' }] }; } //使用刚刚指定的配置项和数据项显示图表 myChart.setOption(option); /* myChart.on('click', function (params) { let trlist = $("#body").children('tr'); var name = params.name; alert("选中"+name+"!查询成功"); for (var i=0;i<trlist.length;i++) { var tdArr = trlist.eq(i).find("td"); var namec = tdArr.eq(0).text();// 备注 if(namec==name){ trlist.eq(i).css("background-color","red"); }else { trlist.eq(i).css("background-color","#E8F6FF"); } } trlist.eq(0).focus(); }); */ myChart.on('click', function (params) { var name = params.name; alert("选中"+name+"!查询成功"); var trlist = $("#body").children('tr'); for (var i=0;i<trlist.length;i++) { var tdArr = trlist.eq(i).find("td"); var namec = tdArr.eq(0).text();// 备注 if(namec==name){ trlist.eq(i).css("background-color","red"); }else { trlist.eq(i).css("background-color","#E8F6FF"); } } trlist.eq(0).focus(); }); </script> </html>
数据传递的Servlet类
package com.Java.Servlet; import java.io.IOException; import java.util.ArrayList; import java.util.List; import javax.servlet.ServletException; import javax.servlet.annotation.WebServlet; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.Java.Bean.info; import com.Java.Dao.dao; import com.google.gson.Gson; /** * Servlet implementation class getinfo */ @WebServlet("/getinfo") public class getinfo extends HttpServlet { private static final long serialVersionUID = 1L; /** * @see HttpServlet#HttpServlet() */ public getinfo() { super(); // TODO Auto-generated constructor stub } /** * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub response.getWriter().append("Served at: ").append(request.getContextPath()); } /** * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) */ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub request.setCharacterEncoding("utf-8"); response.setCharacterEncoding("utf-8"); String tblx=request.getParameter("tblx"); String type=request.getParameter("type"); List<info> list= dao.getinfo(tblx,type); Gson gson= new Gson(); String json=gson.toJson(list); //System.out.println(json); request.setAttribute("json", json); request.setAttribute("info", list); request.setAttribute("type", tblx); request.getRequestDispatcher("test.jsp").forward(request, response); } }
请求数据的dao层
package com.Java.Dao; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import com.Java.Bean.info; import com.Java.Dbutil.DBUtil; public class dao { private static Connection conn=DBUtil.getConn(); @SuppressWarnings("null") public static List<info> getinfo(String tblx, String tblx2) { String sql=null; List<info> list=new ArrayList<info>(); if(tblx2.equals("addr")) { sql="select * from iptu order by sum desc limit 0,5"; }else if(tblx2.equals("cishu")) { sql="select * from traffictu order by sum desc limit 0,5"; }else { sql="select * from typetu order by sum desc limit 0,5"; } System.out.println(sql); try { ResultSet rs=conn.prepareStatement(sql).executeQuery(); while(rs.next()) { info i=new info(); i.setInfo(rs.getString("name")); i.setNum(rs.getString("sum")); //System.out.println(rs.getString("name")); //System.out.println(rs.getString("sum")); list.add(i); } } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } return list; } }
连接数据库的 DBUtil
package com.Java.Dbutil; import java.beans.Statement; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; /** * <p>Title: DBUtil.java</p> * <p>Description: </p> * <p>Copyright: Copyright (c) 2020</p> * @author CuiXingYu * @date 2020年10月5日 * @version 1.0 */ public class DBUtil { // 连接数据库 url路径和用户名 密码 public static String db_url="jdbc:mysql://localhost:3306/homework?serverTimezone=UTC&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true"; public static String db_user="root"; public static String db_password="101032"; public static Connection getConn() { Connection conn=null; try { // 数据库驱动加载 Class.forName("com.mysql.jdbc.Driver"); try { //链接数据库 conn=DriverManager.getConnection(db_url, db_user, db_password); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("驱动链接加载成功！"); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } //返回链接 return conn; } //测试主函数 利用Java运行来测试 @SuppressWarnings("static-access") public static void main(String[] args) throws SQLException { DBUtil dbu=new DBUtil(); Connection conn= dbu.getConn(); /* String sql="insert into student values (?,?,?,?)"; PreparedStatement pstmt=conn.prepareStatement(sql); pstmt.setString(1, "scofield"); pstmt.setInt(2, 45); pstmt.setInt(3, 89); pstmt.setInt(4, 100); pstmt.executeUpdate(); String sql1="select name,English from student where name = 'scofield'"; PreparedStatement preparedStatement=conn.prepareStatement(sql1); ResultSet rs=preparedStatement.executeQuery(); if(rs.next()) { System.out.println(rs.getString("name")); System.out.println(rs.getInt("English")); } */ } //关闭函数 public static void close(Statement state,Connection conn) { //只有状态和连接时，先关闭状态 if(state!=null) { try { ((Connection) state).close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(conn!=null) { try { conn.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } //关闭函数 public static void close(PreparedStatement state,Connection conn) { if(state!=null) { try { ((Connection) state).close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(conn!=null) { try { conn.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } public static void close(ResultSet rs,Statement state,Connection conn) { if(rs!=null) { try { rs.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(state!=null) { try { ((Connection) state).close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(conn!=null) { try { conn.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } //关闭函数 public static void close(java.sql.Statement state, Connection conn) { // TODO Auto-generated method stub if(state!=null) { try { state.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(conn!=null) { try { conn.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } /** * <p>Title: close</p> * <p>Description: </p> * @param connection * @param preparedStatement * @param rs */ public static void close(Connection connection, PreparedStatement preparedStatement, ResultSet rs) { // TODO Auto-generated method stub if(connection!=null) { try { connection.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(preparedStatement!=null) { try { preparedStatement.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(rs!=null) { try { rs.close(); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } }
获取数据的类型 JAVABean
package com.Java.Bean; public class info { private String info; private String num; public String getInfo() { return info; } public void setInfo(String info) { this.info = info; } public String getNum() { return num; } public void setNum(String num) { this.num = num; } }
四丶运行截图
实现效果是可以根据需要查看的数据类型和想要显示的结果对显示的图标类型进行调整（目前只有 柱状图，折线图，饼图三种）
当点击图形当中显示的某一部分数据之后，会随点击事件将其下方遍历的相关数据标红，这是我个人认为的图表联动。
折线图 与 ip 类型
柱状图 与访问次数 类型
饼状图与数据类型
