- 确认报表统计方案
- 统计维度
- 数据收集
- 数据展示
- 复杂SQL的编写,Echarts报表的使用
报表统计方案
- 统计维度:
- 统计数值:总阅读数,总点赞数,今日阅读数,今日点赞数,今日预计阅读数,今日预计阅读增长
- 统计报表:30天阅读/点赞趋势图、文档阅读量排名(热门文章)、文档点赞量排名(优质文章)
- 业务表统计:所有报表数据都是从业务表直接获取的
- 优点:实时性好(数据准确)、工作量
- 缺点:对业务表性能有影响,有些统计无法实现(比如上面的今日阅读数,点赞数就没法统计)
- 中间表统计:定时将业务表数据汇总到中间表,报表数据从中间表获取
- 优点:性能好,可实现多功能统计
- 缺点:工作量大,步骤多容易出错
电子书快照表设计
- 概念:快照
- 电子书快照表:一天存储一次快照
因为我们这个需要每天的统计数据,所以我们的快照是一天一次,也就是说每天要存储一次(条)当时的电子书的信息,我们下面就是电子书快照表的设计sql:
drop table if exists `ebook_snapshot`;
create table `ebook_snapshot` (
`id` bigint auto_increment not null comment 'id',
`ebook_id` bigint not null default 0 comment '电子书id',
`date` date not null comment '快照日期',
`view_count` int not null default 0 comment '阅读数',
`vote_count` int not null default 0 comment '点赞数',
`view_increase` int not null default 0 comment '阅读增长',
`vote_increase` int not null default 0 comment '点赞增长',
primary key (`id`),
unique key `ebook_id_date_unique` (`ebook_id`,`date`)
) engine=innodb default charset=utf8mb4 comment='电子书快照表'
这样的设计保证了每一天每一门电子书应该有一条数据,同时使用ebook_id和date作为唯一键,然后去gennerator-config.xml当中去最后添加下面的代码,并执行mybatis-gennerator生成持久层代码:
<table tableName="ebook_snapshot"/>
电子书快照收集脚本编写
- 从业务表收集数据的SQL尽量简单,不要影响业务表性能
- 快照分成两个部分
- 总量:总阅读数,总点赞数(总量简单,只需要从业务表直接把当前最新的数据把它放进来就可以了)
- 增量:今日阅读数,今日点赞数(稍微复杂一些,把今天的总量减去昨天的总量,就是今天的增量)
我们的收集策略也不能简单的就是在23:59:59这个时间点去收集,而是应该每个小时收集一次,然后每天的第一次收集向表当中插入数据,后续的都来更新这条数据的值即可。所以我们应该先写insert,然后再update,需要更新的字段其中view_count和vote_count只需要从业务表当中取最新的数据即可,而view_increase和vote_increase需要一点计算,需要获取到昨天的view_count、vote_count和今天的view_count、vote_count,然后做差值。
下面我们来书写一下跑批的sql的四个步骤:
- 第一步,为所有电子书生成一条今天的记录,如果还没有
- 第二步,把今天的阅读数、点赞数全部更新进来
- 第三步,根据今天和昨天的总数据来计算出这个阅读的增长
对应的sql如下:
# 插入记录(如果今天没有数据的话)
insert into ebook_snapshot(ebook_id, `date`, view_count, vote_count, view_increase, vote_increase)
select t1.id, curdate(), 0, 0, 0, 0
from ebook t1
where not exists(select 1
from ebook_snapshot t2
where t1.id = t2.ebook_id
and t2.`date` = curdate());
# 更新view_count和vote_count
update ebook_snapshot t1, ebook t2
set t1.view_count = t2.view_count,t1.vote_count = t2.vote_count
where t1.`date`=curdate()
and t1.ebook_id = t2.id;
#获取昨天的数据,两者做差
update ebook_snapshot t1 left join (select ebook_id, view_count, vote_count from ebook_snapshot
where `date` = date_sub(curdate(), interval 1 day)) t2
on t1.ebook_id=t2.ebook_id
set t1.view_increase = (t1.view_count-ifnull(t2.view_count, 0)),
t1.vote_increase = (t1.vote_count-ifnull(t2.vote_count, 0))
where t1.`date` = curdate();
完成电子书快照任务
首先如果我们要在mybatis当中执行多个sql的话,我们需要在数据库的配置当中添加allowMultiQueries=true这样的参数,如下:
# 增加数据库连接
spring.datasource.url=jdbc:mysql://rm-hp3f05153598ck3e5vo.mysql.huhehaote.rds.aliyuncs.com/wikidev?characterEncoding=UTF8&autoReconnect=true&serverTimezone=Asia/Shanghai&allowMultiQueries=true
然后我们就开始倒着展示代码:
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.taopoppy.wiki.mapper.EbookSnapshotMapperCust" >
<!--
# 方案一(ID不连续):
# 删除今天的数据
# 为所有的电子书生成一条今天的记录
# 更新总阅读数、总点赞数
# 更新今日阅读数、今日点赞数
# 方案二(ID连续):
# 为所有的电子书生成一条今天的记录,如果还没有
# 更新总阅读数、总点赞数
# 更新今日阅读数、今日点赞数
-->
<update id="genSnapshot">
insert into ebook_snapshot(ebook_id, `date`, view_count, vote_count, view_increase, vote_increase)
select t1.id, curdate(), 0, 0, 0, 0
from ebook t1
where not exists(select 1
from ebook_snapshot t2
where t1.id = t2.ebook_id
and t2.`date` = curdate());
update ebook_snapshot t1, ebook t2
set t1.view_count = t2.view_count,
t1.vote_count = t2.vote_count
where t1.`date` = curdate()
and t1.ebook_id = t2.id;
update ebook_snapshot t1 left join (select ebook_id, view_count, vote_count
from ebook_snapshot
where `date` = date_sub(curdate(), interval 1 day)) t2
on t1.ebook_id = t2.ebook_id
set t1.view_increase = (t1.view_count - ifnull(t2.view_count, 0)),
t1.vote_increase = (t1.vote_count - ifnull(t2.vote_count, 0))
where t1.`date` = curdate();
</update>
</mapper>
package com.taopoppy.wiki.mapper;
public interface EbookSnapshotMapperCust {
public void genSnapshot();
}
package com.taopoppy.wiki.service;
import com.taopoppy.wiki.mapper.EbookSnapshotMapperCust;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@Service
public class EbookSnapshotService {
@Resource
private EbookSnapshotMapperCust ebookSnapshotMapperCust;
public void genSnapshot() {
ebookSnapshotMapperCust.genSnapshot();
}
}
最后我们在定时器当中去执行快照的程序:
package com.taopoppy.wiki.job;
import com.taopoppy.wiki.service.EbookSnapshotService;
import com.taopoppy.wiki.util.SnowFlake;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@Component
public class EbookSnapshotJob {
private static final Logger LOG = LoggerFactory.getLogger(EbookSnapshotJob.class);
@Resource
private EbookSnapshotService ebookSnapshotService;
@Resource
private SnowFlake snowFlake;
/**
* 自定义cron表达式跑批
* 只有等上一次执行完成,下一次才会在下一个时间点执行,错过就错过
*/
@Scheduled(cron = "0 0/1 * * * ?")
public void doSnapshot() {
// 增加日志流水号
MDC.put("LOG_ID", String.valueOf(snowFlake.nextId()));
LOG.info("生成今日电子书快照开始");
Long start = System.currentTimeMillis();
ebookSnapshotService.genSnapshot();
LOG.info("生成今日电子书快照结束,耗时:{}毫秒", System.currentTimeMillis() - start);
}
}
首页统计功能
1. 后端代码
首页包含数值统计和30天趋势图,所以我们从controller开始看看后端代码:
@RestController
@RequestMapping("/ebook-snapshot")
public class EbookSnapshotController {
@Resource
private EbookSnapshotService ebookSnapshotService;
@GetMapping("/get-statistic")
public CommonResp getStatistic() {
List<StatisticResp> statisticResp = ebookSnapshotService.getStatistic();
CommonResp<List<StatisticResp>> commonResp = new CommonResp<>();
commonResp.setContent(statisticResp);
return commonResp;
}
@GetMapping("/get-30-statistic")
public CommonResp get30Statistic() {
List<StatisticResp> statisticResp = ebookSnapshotService.get30Statistic();
CommonResp<List<StatisticResp>> commonResp = new CommonResp<>();
commonResp.setContent(statisticResp);
return commonResp;
}
}
@Service
public class EbookSnapshotService {
@Resource
private EbookSnapshotMapperCust ebookSnapshotMapperCust;
public void genSnapshot() {
ebookSnapshotMapperCust.genSnapshot();
}
/**
* 获取首页数值数据:总阅读数、总点赞数、今日阅读数、今日点赞数、今日预计阅读数、今日预计阅读增长
*/
public List<StatisticResp> getStatistic() {
return ebookSnapshotMapperCust.getStatistic();
}
/**
* 30天数值统计
*/
public List<StatisticResp> get30Statistic() {
return ebookSnapshotMapperCust.get30Statistic();
}
}
public interface EbookSnapshotMapperCust {
public void genSnapshot();
List<StatisticResp> getStatistic();
List<StatisticResp> get30Statistic();
}
package com.taopoppy.wiki.resp;
import com.fasterxml.jackson.annotation.JsonFormat;
import java.util.Date;
public class StatisticResp {
@JsonFormat(pattern="MM-dd", timezone = "GMT+8")
private Date date;
private int viewCount;
private int voteCount;
private int viewIncrease;
private int voteIncrease;
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
public int getViewCount() {
return viewCount;
}
public void setViewCount(int viewCount) {
this.viewCount = viewCount;
}
public int getVoteCount() {
return voteCount;
}
public void setVoteCount(int voteCount) {
this.voteCount = voteCount;
}
public int getViewIncrease() {
return viewIncrease;
}
public void setViewIncrease(int viewIncrease) {
this.viewIncrease = viewIncrease;
}
public int getVoteIncrease() {
return voteIncrease;
}
public void setVoteIncrease(int voteIncrease) {
this.voteIncrease = voteIncrease;
}
@Override
public String toString() {
return "StatisticResp{" +
"date=" + date +
", viewCount=" + viewCount +
", voteCount=" + voteCount +
", viewIncrease=" + viewIncrease +
", voteIncrease=" + voteIncrease +
'}';
}
}
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.taopoppy.wiki.mapper.EbookSnapshotMapperCust" >
<!-- 获取首页数值数据:总阅读数、总点赞数、今日阅读数、今日点赞数、今日预计阅读数、今日预计阅读增长 -->
<select id="getStatistic" resultType="com.taopoppy.wiki.resp.StatisticResp">
select
t1.`date` as `date`,
sum(t1.view_count) as viewCount,
sum(t1.vote_count) as voteCount,
sum(t1.view_increase) as viewIncrease,
sum(t1.vote_increase) as voteIncrease
from
ebook_snapshot t1
where
t1.`date` >= date_sub(curdate(), interval 1 day)
group by
t1.`date`
order by
t1.`date` asc;
</select>
<select id="get30Statistic" resultType="com.taopoppy.wiki.resp.StatisticResp">
select
t1.`date` as `date`,
sum(t1.view_increase) as viewIncrease,
sum(t1.vote_increase) as voteIncrease
from
ebook_snapshot t1
where
t1.`date` between date_sub(curdate(), interval 30 day) and date_sub(curdate(), interval 1 day)
group by
t1.`date`
order by
t1.`date` asc;
</select>
</mapper>
2. 前端代码
前端是需要去使用echarts的,所以我们可以将echarts.min.js直接拷到项目当中去,然后在index.html去引入即可,具体的统计页面我们写在了the-welcome.vue当中,代码如下:
<template>
<div>
<div class="tip">
<div><b>示例网站说明(网站实现的功能课程都有手把手教):</b></div>
<div>1. 统计数据是真实的,一分钟左右延时,<b>用到了定时器、复杂SQL统计、echarts</b></div>
<div>2. 有文档被别人点赞,你也会收到实时通知哦!<b>用到了websocket、异步化、RocketMQ、防重设计</b></div>
<div>3. 文档树可无限级扩展,支持文字、图片、视频。<b>用到了无限级树设计知识</b></div>
<div>4. 登录后可看到更多菜单。<b>用到了单点登录、前后端登录拦截、安全性设计</b></div>
<div>课程技术栈思维导图,
<a href="http://www.jiawablog.com/detail?id=152412053087326208" target="_blank">
点击查看:SpringBoot+Vue3
</a>
</div>
<div>你也想有个WIKI知识库吗?,<b>配套视频课程</b>:
<a href="https://coding.imooc.com/class/474.html" target="_blank">
《SpringBoot知识体系+Vue3全家桶 前后端分离 实战WIKI知识库系统》
</a>
</div>
</div>
<a-row>
<a-col :span="24">
<a-card>
<a-row>
<a-col :span="8">
<a-statistic title="总阅读量" :value="statistic.viewCount">
<template #suffix>
<UserOutlined />
</template>
</a-statistic>
</a-col>
<a-col :span="8">
<a-statistic title="总点赞量" :value="statistic.voteCount">
<template #suffix>
<like-outlined />
</template>
</a-statistic>
</a-col>
<a-col :span="8">
<a-statistic title="点赞率" :value="statistic.voteCount / statistic.viewCount * 100"
:precision="2"
suffix="%"
:value-style="{ color: '#cf1322' }">
<template #suffix>
<like-outlined />
</template>
</a-statistic>
</a-col>
</a-row>
</a-card>
</a-col>
</a-row>
<br>
<a-row :gutter="16">
<a-col :span="12">
<a-card>
<a-row>
<a-col :span="12">
<a-statistic title="今日阅读" :value="statistic.todayViewCount" style="margin-right: 50px">
<template #suffix>
<UserOutlined />
</template>
</a-statistic>
</a-col>
<a-col :span="12">
<a-statistic title="今日点赞" :value="statistic.todayVoteCount">
<template #suffix>
<like-outlined />
</template>
</a-statistic>
</a-col>
</a-row>
</a-card>
</a-col>
<a-col :span="12">
<a-card>
<a-row>
<a-col :span="12">
<a-statistic
title="预计今日阅读"
:value="statistic.todayViewIncrease"
:value-style="{ color: '#0000ff' }"
>
<template #suffix>
<UserOutlined />
</template>
</a-statistic>
</a-col>
<a-col :span="12">
<a-statistic
title="预计今日阅读增长"
:value="statistic.todayViewIncreaseRateAbs"
:precision="2"
suffix="%"
class="demo-class"
:value-style="statistic.todayViewIncreaseRate < 0 ? { color: '#3f8600' } : { color: '#cf1322' }"
>
<template #prefix>
<arrow-down-outlined v-if="statistic.todayViewIncreaseRate < 0"/>
<arrow-up-outlined v-if="statistic.todayViewIncreaseRate >= 0"/>
</template>
</a-statistic>
</a-col>
</a-row>
</a-card>
</a-col>
</a-row>
<br>
<a-row>
<a-col :span="24" id="main-col">
<div id="main" style="width: 100%;height:300px;"></div>
</a-col>
</a-row>
</div>
</template>
<script lang="ts">
import { defineComponent, ref, onMounted } from 'vue'
import axios from 'axios';
declare let echarts: any;
export default defineComponent({
name: 'the-welcome',
setup () {
const statistic = ref();
statistic.value = {};
const getStatistic = () => {
axios.get('/ebook-snapshot/get-statistic').then((response) => {
const data = response.data;
if (data.success) {
const statisticResp = data.content;
statistic.value.viewCount = statisticResp[1].viewCount;
statistic.value.voteCount = statisticResp[1].voteCount;
statistic.value.todayViewCount = statisticResp[1].viewIncrease;
statistic.value.todayVoteCount = statisticResp[1].voteIncrease;
// 按分钟计算当前时间点,占一天的百分比
const now = new Date();
const nowRate = (now.getHours() * 60 + now.getMinutes()) / (60 * 24);
// console.log(nowRate)
statistic.value.todayViewIncrease = parseInt(String(statisticResp[1].viewIncrease / nowRate));
// todayViewIncreaseRate:今日预计增长率
statistic.value.todayViewIncreaseRate = (statistic.value.todayViewIncrease - statisticResp[0].viewIncrease) / statisticResp[0].viewIncrease * 100;
statistic.value.todayViewIncreaseRateAbs = Math.abs(statistic.value.todayViewIncreaseRate);
}
});
};
const init30DayEcharts = (list: any) => {
// 发布生产后出现问题:切到别的页面,再切回首页,报表显示不出来
// 解决方法:把原来的id=main的区域清空,重新初始化
const mainDom = document.getElementById('main-col');
if (mainDom) {
mainDom.innerHTML = '<div id="main" style="width: 100%;height:300px;"></div>';
}
// 基于准备好的dom,初始化echarts实例
const myChart = echarts.init(document.getElementById('main'));
const xAxis = [];
const seriesView = [];
const seriesVote = [];
for (let i = 0; i < list.length; i++) {
const record = list[i];
xAxis.push(record.date);
seriesView.push(record.viewIncrease);
seriesVote.push(record.voteIncrease);
}
// 指定图表的配置项和数据
const option = {
title: {
text: '30天趋势图'
},
tooltip: {
trigger: 'axis'
},
legend: {
data: ['总阅读量', '总点赞量']
},
grid: {
left: '1%',
right: '3%',
bottom: '3%',
containLabel: true
},
toolbox: {
feature: {
saveAsImage: {}
}
},
xAxis: {
type: 'category',
boundaryGap: false,
data: xAxis
},
yAxis: {
type: 'value'
},
series: [
{
name: '总阅读量',
type: 'line',
// stack: '总量', 不堆叠
data: seriesView,
smooth: true
},
{
name: '总点赞量',
type: 'line',
// stack: '总量', 不堆叠
data: seriesVote,
smooth: true
}
]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
};
const get30DayStatistic = () => {
axios.get('/ebook-snapshot/get-30-statistic').then((response) => {
const data = response.data;
if (data.success) {
const statisticList = data.content;
init30DayEcharts(statisticList)
}
});
};
const testEcharts = () => {
// 基于准备好的dom,初始化echarts实例
const myChart = echarts.init(document.getElementById('main'));
// 指定图表的配置项和数据
const option = {
title: {
text: 'ECharts 入门示例'
},
tooltip: {},
legend: {
data:['销量']
},
xAxis: {
data: ["衬衫","羊毛衫","雪纺衫","裤子","高跟鞋","袜子"]
},
yAxis: {},
series: [{
name: '销量',
type: 'bar',
data: [5, 20, 36, 10, 10, 20]
}]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
};
onMounted(() => {
getStatistic();
// testEcharts();
get30DayStatistic();
});
return {
statistic
}
}
});
</script>
<style scoped>
.tip {
padding: 10px 5px;
margin-bottom: 20px;
border: 1px solid transparent;
background: linear-gradient(white,white) padding-box,repeating-linear-gradient(-45deg, black 0, black 25%, white 0, white 50%) 0/.6em .6em;
animation:ants 12s linear infinite;
}
.tip b{
color: red;
}
</style>