原文链接

写在前面：

自己已经创建公众号啦~

AI算法交流+开源数据汇总+私房数据及标注数据共享+自己实践项目开源

欢迎大家关注：DeepAI 视界

【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图1
展示一下：
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图2
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图3
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图4
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图5
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图6
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图7
【项目实战】数据爬虫 ＋ 数据清洗 数据可视化 开源代码啦 - 图8

爬虫：链接网二手房（以贵阳市为例）

  1. """
  2. 爬虫
  3. """
  4. import requests
  5. from lxml import etree
  6. import xlwt
  7. import xlrd
  8. import csv
  9. import pandas as pd
  10. import time
  11. import re
  12. class LanjiaSpider:
  13.     def __init__(self):
  14.         self.url = 'https://wh.lianjia.com/ershoufang/ronganxian/pg{}/'
  15.         self.headers = {
  16.             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}
  18.     def get_response_spider(self, url_str):  # 发送请求
  19.         get_response = requests.get(url_str, headers=self.headers)
  20.         time.sleep(2)
  21.         response = get_response.content.decode()
  22.         html = etree.HTML(response)
  23.         return html
  24.     def get_content_html(self, html):  # 使xpath获取数据
  25.         self.houseInfo = html.xpath('//div[@class="houseInfo"]/text()')
  26.         self.title = html.xpath('//div[@class="title"]/a/text()')
  27.         self.positionInfo = html.xpath('//div[@class="positionInfo"]/a/text()')
  28.         self.totalPrice = html.xpath('//div[@class="totalPrice"]/span/text()')
  29.         self.unitPrice = html.xpath('//div[@class="unitPrice"]/span/text()')
  30.         self.followInfo = html.xpath('//div[@class="followInfo"]/text()')
  31.         self.tag = html.xpath('//div[@class="tag"]/span/text()')
  32.         # print(title)
  33.         # return houseInfo,title,positionInfo,totalPrice,unitPrice,followInfo,tag
  35.     def xpath_houseInfo(self):
  36.         #print(self.houseInfo)
  37.         #print(type(self.houseInfo))
  38.         # df = pd.DataFrame({"houseInfo": self.houseInfo,"tite":self.title,"positionInfo":self.positionInfo,"totaPrice":self.totalPrice,"unitPrice":self.unitPrice,"followInfo":self.followInfo,"tag":self.tag})
  39.         # df=pd.DataFrame({"houseInfo": self.houseInfo,"tite":self.title})
  40.         # df.to_excel(r'C:\Users\wy\Desktop\sublime\链家\pand3.xlsx')
  41.         # a=len(self.houseInfo)
  42.         for i in range(len(self.houseInfo)):
  43.             # print(i)
  44.             # yield i
  45.             # print(type(self.houseInfo))
  46.             yield self.houseInfo[i]
  48.     def qingxi_data_houseInfo(self):  # 清洗数据
  50.         self.xpath_houseInfo()
  51.         self.xpath_title()
  52.         self.xpath_positionInfo()
  53.         self.xpath_totalPrice()
  54.         self.xpath_unitPrice()
  55.         self.xpath_followInfo()
  56.         self.xpath_tag()
  57.         get_houseInfo = self.xpath_houseInfo()
  58.         get_title = self.xpath_title()
  59.         get_positionInfo=self.xpath_positionInfo()
  60.         get_totalPrice = self.xpath_totalPrice()
  61.         get_unitPrice = self.xpath_unitPrice()
  62.         get_followInfo=self.xpath_followInfo()
  63.         get_tag=self.xpath_tag()
  64.         i = 1
  66.         while True:
  67.             data_houseInfo= next(get_houseInfo)
  68.             data_title=next(get_title)
  69.             data_positionInfo=next(get_positionInfo)
  70.             data_totalPrice=next(get_totalPrice)
  71.             data_unitPrice=next(get_unitPrice)
  72.             data_followInfo=next(get_followInfo)
  73.             data_tag=next(get_tag)
  75.             with open("a.csv", "a", newline="", encoding="utf-8-sig") as f:
  76.                 # fieldnames = ['houseInfo', 'title', 'positionInfo', 'totalPrice/万元', 'unitPrice', 'followInfo', 'tag']
  77.                 # writer = csv.DictWriter(f, fieldnames=fieldnames)  # 写入表头
  78.                 # writer.writeheader()
  79.                 writer = csv.DictWriter(f, fieldnames=fieldnames)  # 写入表头
  81.                 list_1 = ['houseInfo', 'title', 'positionInfo', 'totalPrice/万元', 'unitPrice', 'followInfo', 'tag']
  82.                 list_2 = [data_houseInfo,data_title,data_positionInfo,data_totalPrice,data_unitPrice,data_followInfo,data_tag]
  83.                 list_3 = dict(zip(list_1, list_2))
  84.                 writer.writerow(list_3)
  85.                 print("写入第"+str(i)+"行数据")
  86.             i += 1
  87.             if i > len(self.houseInfo):
  88.                 break
  90.     def xpath_title(self):
  91.         for i in range(len(self.title)):
  92.             yield self.title[i]
  94.     def xpath_positionInfo(self):
  95.         for i in range(len(self.positionInfo)):
  96.             yield self.positionInfo[i]
  98.     def xpath_totalPrice(self):
  99.         for i in range(len(self.totalPrice)):
  100.             yield self.totalPrice[i]
  101.     def xpath_unitPrice(self):
  102.         for i in range(len(self.unitPrice)):
  103.             yield self.unitPrice[i]
  104.     def xpath_followInfo(self):
  105.         for i in range(len(self.followInfo)):
  106.             yield self.followInfo[i]
  107.     def xpath_tag(self):
  108.         for i in range(len(self.tag)):
  109.             yield self.tag[i]
  110.     def run(self):
  111.         i = 1
  112.         while True:
  113.             url_str = self.url.format(i)  # 构造请求url
  114.             html = self.get_response_spider(url_str)
  115.             self.get_content_html(html)
  116.             self.qingxi_data_houseInfo()
  118.             i += 1
  119.             if i == 1:  # 不包括57页
  120.                 break
  123. if __name__ == "__main__":
  124.     with open("a.csv", "a", newline="", encoding="utf-8-sig") as f:
  125.         fieldnames = ['houseInfo', 'title', 'positionInfo', 'totalPrice/万元', 'unitPrice', 'followInfo', 'tag']
  126.         writer = csv.DictWriter(f, fieldnames=fieldnames)  # 写入表头
  127.         writer.writeheader()
  128.     lanjia = LanjiaSpider()
  129.     lanjia.run()

对应的数据可视化：

  1. """
  2. 数据分析及可视化
  3. """
  4. import pandas as pd
  5. from pyecharts.charts import Line, Bar
  6. import numpy as np
  7. from pyecharts.globals import ThemeType
  8. from pyecharts.charts import Pie
  9. from pyecharts import options as opts
  12. places = ['lianjia_BaiYunQu', 'lianjia_GuanShanHuQu', 'lianjia_HuaXiQu', 'lianjia_NanMingQu', 'lianjia_WuDangQu', 'lianjia_YunYanQu']
  13. place = ['白云区', '观山湖区', '花溪区', '南明区', '乌当区', '云岩区']
  14. avgs = []  # 房价均值
  15. median = []  # 房价中位数
  16. favourate_avg = []  # 房价收藏人数均值
  17. favourate_median = []  # 房价收藏人数中位数
  18. houseidfo = ['2室1厅', '3室1厅', '2室2厅', '3室2厅', '其他']  # 房型定义
  19. houseidfos = ['2.1', '3.1', '2.2', '3.2']
  20. sum_house = [0,  0, 0, 0, 0]  # 各房型数量
  21. price = []  # 房价
  22. fav = []  # 收藏人数
  23. type = []
  24. area = []  # 房间面积
  25. def avg(name):
  26.     df = pd.read_csv(str(name)+'.csv', encoding='utf-8')
  27.     pattern = '\d+'
  28.     df['totalPrice/万元'] = df['totalPrice/万元'].str.findall(pattern)
  29.     df['followInfo'] = df['followInfo'].str.findall(pattern)
  30.     df['houseInfo'] = df['houseInfo'].str.findall(pattern)
  31.     sum_houses = [0, 0, 0, 0, 0]
  32.     # print(sum_house)
  33.     avg_work_year = []
  34.     medians = []
  35.     favourates = []
  36.     k = 0
  37.     k1 = 0
  38.     k3 = 0
  39.     k4 = 0
  41.     for i in range(len(df)):
  42.         if (i + 1) % 2 == 0:
  43.             continue
  44.         else:
  45.             if len(df['totalPrice/万元'][i]) == 2:
  46.                 avg_work_year.append(','.join(df['totalPrice/万元'][i]).replace(',', '.'))
  47.                 medians.append(float(','.join(df['totalPrice/万元'][i]).replace(',', '.')))
  48.                 price.append(','.join(df['totalPrice/万元'][i]).replace(',', '.'))
  49.             if len(df['followInfo'][i]) ==2:
  50.                 favourates.append(int(','.join(df['followInfo'][i][:1])))
  51.                 fav.append(int(','.join(df['followInfo'][i][:1])))
  52.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 2.1:
  53.                 k +=1
  54.                 sum_houses[0] =k
  55.                 type.append(2.1)
  56.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 3.1:
  57.                 k1 +=1
  58.                 sum_houses[1] =k1
  59.                 type.append(3.1)
  60.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 2.2:
  61.                 k3 +=1
  62.                 sum_houses[2] =k3
  63.                 type.append(2.2)
  64.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 3.2:
  65.                 k4 +=1
  66.                 sum_houses[3] =k4
  67.                 type.append(3.2)
  68.             else:
  69.                 k4 +=1
  70.                 sum_houses[4] = k4
  71.                 type.append('other')
  72.             area.append(float(','.join(df['houseInfo'][i][2:4]).replace(',', '.')))
  73.     sum_house[0] =sum_houses[0]
  74.     sum_house[1] = sum_houses[1]
  75.     sum_house[2] = sum_houses[2]
  76.     sum_house[3] = sum_houses[3]
  77.     sum_house[4] = sum_houses[4]
  79.     favourates.sort()
  80.     favourate_median.append(int(np.median(favourates)))
  81.     medians.sort()
  82.     median.append(np.median(medians))
  83.     # price = avg_work_year
  84.     b = len(avg_work_year)
  85.     b1= len(favourates)
  87.     sum = 0
  88.     sum1 = 0
  89.     for i in avg_work_year:
  90.         sum = sum+float(i)
  91.     avgs.append(round(sum/b, 2))
  92.     for i in favourates:
  93.         sum1 = sum1+float(i)
  94.     favourate_avg.append(round(int(sum1/b1), 2))
  96. for i in places:
  97.     avg(i)
  99. print("各区平均房价", avgs)
  100. print('各房型的出售总数:', sum_house)
  101. print("房间面积", area)
  102. """
  103. [280, 56, 504, 1676, 1680]
  104. [392, 112, 448, 1679, 1680]
  105. [224, 0, 616, 3359, 3360]
  106. [448, 112, 280, 1679, 1680]
  107. [504, 0, 336, 1680, 1679]
  108. [224, 56, 168, 1680, 1670]
  111. [66.17, 65.6, 76.04, 78.94, 62.06, 74.37] 
  112. [68.8, 67.8, 79.8, 70.8, 57.6, 78.8]
  113. [6, 6, 9, 4, 4, 4] [5, 4, 3, 2, 3, 2]
  114. """
  115. # print(median)
  116. # print(favourate_avg,favourate_median)
  117. line = Line()
  118. line.add_xaxis(place)
  119. line.add_yaxis("贵阳各地房价平均值(万元)", avgs)
  120. line.add_yaxis("贵阳各地房价中位数值（万元）", median)
  121. # line.render("predict_line.html")
  123. def bar() -> Bar:
  124.     c = (
  125.         Bar({"theme": ThemeType.MACARONS})
  126.             .add_xaxis(place)
  127.             .add_yaxis("平均值", avgs)
  128.             .add_yaxis("中位数", median)
  129.             .set_global_opts(
  130.             title_opts={"text": "贵阳各地房价(万元)"}
  131.         )
  132.     )
  133.     return c
  134. bar().render("predict_bar.html")
  135. # print(sum_house)
  136. def bar() -> Bar:
  137.     c = (
  138.         Bar({"theme": ThemeType.MACARONS})
  139.             .add_xaxis(houseidfo)
  140.             .add_yaxis(place[0], [280, 56, 504, 1676, 1680])
  141.             .add_yaxis(place[1], [392, 112, 448, 1679, 1680])
  142.             .add_yaxis(place[2], [224, 0, 616, 3359, 3360])
  143.             .add_yaxis(place[3], [448, 112, 280, 1679, 1680])
  144.             .add_yaxis(place[4], [504, 0, 336, 1680, 1679])
  145.             .add_yaxis(place[-1], sum_house)
  146.             # .add_yaxis("中位数", favourate_median)
  147.             .set_global_opts(
  148.             title_opts={"text": "贵阳各地房型\n数量"}
  149.         )
  150.     )
  151.     return c
  152. # bar().render("house_bar.html")
  153. line = Line()
  154. line.add_xaxis(place)
  155. line.add_yaxis("贵阳各地房子平均面积\n(平米)", area)
  156. line.render("Area_line.html")
  160. list_num = favourate_avg
  161. attr = place
  162. # print(zip(attr, list_num))
  163. s = [list(z) for z in zip(attr, list_num)]
  164. c = (Pie().add("", s).set_global_opts(title_opts=opts.TitleOpts(title="贵阳市各区楼房\n平均收藏人数"))
  165.      .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
  166.      )
  167. c.render("pie_avg.html")
  169. list_num = favourate_median
  170. attr = place
  171. # print(zip(attr, list_num))
  172. s = [list(z) for z in zip(attr, list_num)]
  173. c = (Pie().add("", s).set_global_opts(title_opts=opts.TitleOpts(title="贵阳市各区楼房\n收藏人数中位数"))
  174.      .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
  175.      )
  176. c.render("pie_median.html")
  178. from pyecharts import options as  opts
  179. from pyecharts.charts import Scatter3D
  180. from pyecharts.faker import Faker
  183. price=[float(i)/1 for i in price]
  184. # print(price)
  185. # types=list(map(mapfunc,df.house_type.values))
  186. # type = [224, 56, 168, 1680, 1670]
  187. data = []
  188. # print(fav,type)
  189. # for j in range(len(type)):
  190. #     for k in range(len(fav)):
  191. for j in range(100):
  192.     for k in range(100):
  193.         for i in range(500):
  194.             try:
  195.                 data.append([type[j], favourate_avg[k],price[i]])
  196.             except:
  197.                 continue
  198. # print(data)
  199. scatter = (
  200.      Scatter3D(init_opts=opts.InitOpts(width='900px', height='600px'))  # 初始化
  201.           .add("", data,
  202.                grid3d_opts=opts.Grid3DOpts(
  203.                     width=300, depth=300, rotate_speed=300, is_rotate=True,
  204.                ),)
  206.           # 设置全局配置项
  207.           .set_global_opts(
  208.           title_opts=opts.TitleOpts(title="房型——关注度——价格\n三维关系图"),  # 添加标题
  209.           visualmap_opts=opts.VisualMapOpts(
  210.                max_=100,  # 最大值
  211.                pos_top=200,  # visualMap 组件离容器上侧的距离
  212.                range_color=Faker.visual_color  # 颜色映射
  213.           )
  214.      )
  215.           # .render("3D散点图.html")
  216. )
  217. print('数据分析和可视化结束，左边点开~')

同时赠送给大家另一个版本的：

爬虫：链家网：柳州市

  1. # -*- coding: utf-8 -*-
  2. import scrapy
  5. import requests
  6. from lxml import etree
  7. import xlwt
  8. import xlrd
  9. import csv
  10. import pandas as pd
  11. import time
  13. class LanjiaSpider:
  14.     def __init__(self):
  15.         self.url = 'https://liuzhou.lianjia.com/ershoufang/yufengqu/pg{}/'
  16.         self.headers = {
  17.             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}
  19.     def get_response_spider(self, url_str):  # 发送请求
  20.         get_response = requests.get(self.url, headers=self.headers)
  21.         time.sleep(2)
  22.         response = get_response.content.decode()
  23.         html = etree.HTML(response)
  24.         return html
  25.     def get_content_html(self, html):  # 使xpath获取数据
  26.         self.houseInfo = html.xpath('//div[@class="houseInfo"]/text()')
  27.         self.title = html.xpath('//div[@class="title"]/a/text()')
  28.         self.positionInfo = html.xpath('//div[@class="positionInfo"]/a/text()')
  29.         self.totalPrice = html.xpath('//div[@class="totalPrice"]/span/text()')
  30.         self.unitPrice = html.xpath('//div[@class="unitPrice"]/span/text()')
  31.         self.followInfo = html.xpath('//div[@class="followInfo"]/text()')
  32.         self.tag = html.xpath('//div[@class="tag"]/span/text()')
  33.         # print(title)
  34.         # return houseInfo,title,positionInfo,totalPrice,unitPrice,followInfo,tag
  36.     def xpath_houseInfo(self):
  37.         #print(self.houseInfo)
  38.         #print(type(self.houseInfo))
  39.         # df = pd.DataFrame({"houseInfo": self.houseInfo,"tite":self.title,"positionInfo":self.positionInfo,"totaPrice":self.totalPrice,"unitPrice":self.unitPrice,"followInfo":self.followInfo,"tag":self.tag})
  40.         # df=pd.DataFrame({"houseInfo": self.houseInfo,"tite":self.title})
  41.         # df.to_excel(r'C:\Users\wy\Desktop\sublime\链家\pand3.xlsx')
  42.         # a=len(self.houseInfo)
  43.         for i in range(len(self.houseInfo)):
  44.             # print(i)
  45.             # yield i
  46.             # print(type(self.houseInfo))
  47.             yield self.houseInfo[i]
  49.     def qingxi_data_houseInfo(self):  # 清洗数据
  51.         self.xpath_houseInfo()
  52.         self.xpath_title()
  53.         self.xpath_positionInfo()
  54.         self.xpath_totalPrice()
  55.         self.xpath_unitPrice()
  56.         self.xpath_followInfo()
  57.         self.xpath_tag()
  58.         get_houseInfo = self.xpath_houseInfo()
  59.         get_title = self.xpath_title()
  60.         get_positionInfo=self.xpath_positionInfo()
  61.         get_totalPrice = self.xpath_totalPrice()
  62.         get_unitPrice = self.xpath_unitPrice()
  63.         get_followInfo=self.xpath_followInfo()
  64.         get_tag=self.xpath_tag()
  65.         i = 1
  66.         while True:
  67.             data_houseInfo= next(get_houseInfo)
  68.             data_title=next(get_title)
  69.             data_positionInfo=next(get_positionInfo)
  70.             data_totalPrice=next(get_totalPrice)
  71.             data_unitPrice=next(get_unitPrice)
  72.             data_followInfo=next(get_followInfo)
  73.             data_tag=next(get_tag)
  75.             with open("yufengqu.csv", "a", newline="", encoding="utf-8-sig") as f:
  76.                 fieldnames = ['houseInfo', 'title', 'positionInfo', 'totalPrice/万元', 'unitPrice', 'followInfo', 'tag']
  77.                 writer = csv.DictWriter(f, fieldnames=fieldnames)  # 写入表头
  78.                 writer.writeheader()
  79.                 list_1 = ['houseInfo', 'title', 'positionInfo', 'totalPrice/万元', 'unitPrice', 'followInfo', 'tag']
  80.                 list_2 = [data_houseInfo,data_title,data_positionInfo,data_totalPrice,data_unitPrice,data_followInfo,data_tag]
  81.                 list_3 = dict(zip(list_1, list_2))
  82.                 writer.writerow(list_3)
  83.                 print("写入第"+str(i)+"行数据")
  84.             i += 1
  85.             if i > len(self.houseInfo):
  86.                 break
  88.     def xpath_title(self):
  89.         for i in range(len(self.title)):
  90.             yield self.title[i]
  92.     def xpath_positionInfo(self):
  93.         for i in range(len(self.positionInfo)):
  94.             yield self.positionInfo[i]
  96.     def xpath_totalPrice(self):
  97.         for i in range(len(self.totalPrice)):
  98.             yield self.totalPrice[i]
  99.     def xpath_unitPrice(self):
  100.         for i in range(len(self.unitPrice)):
  101.             yield self.unitPrice[i]
  102.     def xpath_followInfo(self):
  103.         for i in range(len(self.followInfo)):
  104.             yield self.followInfo[i]
  105.     def xpath_tag(self):
  106.         for i in range(len(self.tag)):
  107.             yield self.tag[i]
  108.     def run(self):
  109.         i = 1
  110.         while True:
  111.             url_str = self.url.format(i)  # 构造请求url
  112.             html = self.get_response_spider(url_str)
  113.             self.get_content_html(html)
  114.             self.qingxi_data_houseInfo()
  116.             i += 1
  117.             if i == 100:  # 不包括100页
  118.                 break
  121. # if __name__ == "__main__":
  122. #     lanjia = LanjiaSpider()
  123. #     lanjia.run()
  125. class MyspiderSpider(scrapy.Spider):
  126.     name = 'myspider'
  127.     allowed_domains = ['https://wh.lianjia.com/ershoufang/jianghan/']
  128.     start_urls = ['https://wh.lianjia.com/ershoufang/jianghan//']
  130.     def parse(self, response):
  131.         print('爬取ing....')
  132.         lanjia = LanjiaSpider()
  133.         lanjia.run()

数据可视化：（优化版）

  1. """
  2. 数据分析及可视化
  3. auuthor: 周小夏
  4. """
  5. import pandas as pd
  6. from pyecharts.charts import Line, Bar
  7. import numpy as np
  8. from pyecharts.globals import ThemeType
  9. from pyecharts.charts import Pie
  10. from pyecharts import options as opts
  13. places = ['chengzhongqu', 'liubeiqu', 'liuchengxian', 'liujiangqu', 'liunanqu', 'yufengqu']
  14. place = ['城中区', '柳北区', '柳城县', '柳江区', '柳南区', '鱼峰区']
  15. avgs = []  # 房价均值
  16. median = []  # 房价中位数
  17. favourate_avg = []  # 房价收藏人数均值
  18. favourate_median = []  # 房价收藏人数中位数
  19. houseidfo = ['2室1厅', '3室1厅', '2室2厅', '3室2厅', '其他']  # 房型定义
  20. houseidfos = ['2.1', '3.1', '2.2', '3.2']
  21. sum_house = [0,  0, 0, 0, 0]  # 各房型数量
  22. sum_houses = []
  23. price = []  # 房价均值
  24. unitprice = []  # 单价
  25. fav = []  # 收藏人数
  26. type = []
  27. area = []  # 房间
  30. def avg(name):
  31.     df = pd.read_csv('./spiders/' + str(name)+'.csv', encoding='utf-8')
  32.     pattern = '\d+'
  33.     df['totalPrice/万元'] = df['totalPrice/万元'].str.findall(pattern)
  34.     df['followInfo'] = df['followInfo'].str.findall(pattern)
  35.     df['houseInfo'] = df['houseInfo'].str.findall(pattern)
  36.     df['unitPrice'] = df['unitPrice'].str.findall(pattern)
  38.     sum_houses = [0, 0, 0, 0, 0]
  39.     # print(sum_house)
  40.     avg_work_year = []
  41.     areas = []
  42.     unit_avg = []
  43.     medians = []
  44.     favourates = []
  45.     k = 0
  46.     k1 = 0
  47.     k3 = 0
  48.     k4 = 0
  50.     for i in range(len(df)):
  51.         if (i + 1) % 2 == 0:
  52.             continue
  53.         else:
  54.             if len(df['unitPrice'][i]) >= 0:
  55.                 unit_avg.append(','.join(df['unitPrice'][i]).replace(',', '.'))
  56.             if len(df['totalPrice/万元'][i]) >= 0:
  57.                 avg_work_year.append(','.join(df['totalPrice/万元'][i]).replace(',', '.'))
  58.                 medians.append(float(','.join(df['totalPrice/万元'][i]).replace(',', '.'))*100)
  59.                 price.append(','.join(df['totalPrice/万元'][i]).replace(',', '.'))
  60.             if len(df['followInfo'][i]) ==2:
  61.                 favourates.append(int(','.join(df['followInfo'][i][:1])))
  62.                 fav.append(int(','.join(df['followInfo'][i][:1])))
  63.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 2.1:
  64.                 k +=1
  65.                 sum_houses[0] =k
  66.                 type.append(2.1)
  67.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 3.1:
  68.                 k1 +=1
  69.                 sum_houses[1] =k1
  70.                 type.append(3.1)
  71.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 2.2:
  72.                 k3 +=1
  73.                 sum_houses[2] =k3
  74.                 type.append(2.2)
  75.             if float(','.join(df['houseInfo'][i][:2]).replace(',', '.')) == 3.2:
  76.                 k4 +=1
  77.                 sum_houses[3] =k4
  78.                 type.append(3.2)
  79.             else:
  80.                 k4 +=1
  81.                 sum_houses[4] = k4
  82.                 type.append('other')
  83.             areas.append(float(','.join(df['houseInfo'][i][2:4]).replace(',', '.')))
  84.     sum_house[0] =sum_houses[0]
  85.     sum_house[1] = sum_houses[1]
  86.     sum_house[2] = sum_houses[2]
  87.     sum_house[3] = sum_houses[3]
  88.     sum_house[4] = sum_houses[4]
  89.     sum_house.append(sum_house[0])
  90.     sum_house.append(sum_house[1])
  91.     sum_house.append(sum_house[2])
  92.     sum_house.append(sum_house[3])
  93.     sum_house.append(sum_house[4])
  94.     # print(sum_houses)
  95.     favourates.sort()
  96.     favourate_median.append(int(np.median(favourates)))
  97.     medians.sort()
  98.     median.append(np.median(medians))
  99.     # price = avg_work_year
  100.     b = len(avg_work_year)*100
  101.     b1= len(favourates)
  102.     b2 = len(unit_avg)
  103.     b4 = len(areas)*100
  104.     sum = 0
  105.     sum1 = 0
  106.     for i in unit_avg:
  107.         sum = sum+float(i)
  108.     unitprice.append(round(sum/b2, 2))
  109.     for i in areas:
  110.         sum = sum+float(i)
  111.     area.append(round(sum/b4, 2))
  112.     for i in avg_work_year:
  113.         sum = sum+float(i)
  114.     avgs.append(round(sum/b, 2))
  115.     for i in favourates:
  116.         sum1 = sum1+float(i)
  117.     favourate_avg.append(round(int(sum1/b1), 2))
  119. for i in places:
  120.     avg(i)
  122. print("各区平均房价", avgs)
  123. print('各房型的出售总数:', sum_house)
  124. print("房间面积", area)
  125. print("房价单价", unitprice)
  127. a = []
  128. for i in median:
  129.     a.append(i/100)
  130. # print(median)
  131. # print(favourate_avg,favourate_median)
  132. line = Line()
  133. line.add_xaxis(place)
  134. line.add_yaxis("柳州市各地房价平均值(万元)", avgs)
  135. line.add_yaxis("柳州市各地房价中位数值（万元）", a)
  136. line.render("predict_line.html")
  138. def bar() -> Bar:
  139.     c = (
  140.         Bar({"theme": ThemeType.MACARONS})
  141.             .add_xaxis(place)
  142.             .add_yaxis("平均值", unitprice)
  143.             .set_global_opts(
  144.             title_opts={"text": "柳州市各地房价单价(元)"}
  145.         )
  146.     )
  147.     return c
  149. bar().render("unit_prices.html")
  150. def bar() -> Bar:
  151.     c = (
  152.         Bar({"theme": ThemeType.MACARONS})
  153.             .add_xaxis(place)
  154.             .add_yaxis("平均值", avgs)
  155.             .add_yaxis("中位数", a)
  156.             .set_global_opts(
  157.             title_opts={"text": "柳州市各地房价(万元)"}
  158.         )
  159.     )
  160.     return c
  161. bar().render("predict_bar.html")
  162. # print(sum_house)
  163. def bar() -> Bar:
  164.     c = (
  165.         Bar({"theme": ThemeType.MACARONS})
  166.             .add_xaxis(houseidfo)
  167.             .add_yaxis(place[0], sum_house[0:5])
  168.             .add_yaxis(place[1], sum_house[5:10])
  169.             .add_yaxis(place[2], sum_house[10:15])
  170.             .add_yaxis(place[3], sum_house[15:20])
  171.             .add_yaxis(place[4], sum_house[20:25])
  172.             .add_yaxis(place[-1], sum_house[25:30])
  173.             # .add_yaxis("中位数", favourate_median)
  174.             .set_global_opts(
  175.             title_opts={"text": "柳州市各地房型\n数量"}
  176.         )
  177.     )
  178.     return c
  179. bar().render("house_bar.html")
  181. line = Line()
  182. line.add_xaxis(place)
  183. line.add_yaxis("柳州市各地房子平均面积\n(平米)", area)
  184. line.render("Area_line.html")
  188. list_num = favourate_avg
  189. attr = place
  190. # print(zip(attr, list_num))
  191. s = [list(z) for z in zip(attr, list_num)]
  192. c = (Pie().add("", s).set_global_opts(title_opts=opts.TitleOpts(title="柳州市各区楼房\n平均收藏人数"))
  193.      .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
  194.      )
  195. c.render("pie_avg.html")
  197. list_num = favourate_median
  198. attr = place
  199. # print(zip(attr, list_num))
  200. s = [list(z) for z in zip(attr, list_num)]
  201. c = (Pie().add("", s).set_global_opts(title_opts=opts.TitleOpts(title="柳州市各区楼房\n收藏人数中位数"))
  202.      .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
  203.      )
  204. c.render("pie_median.html")
  206. from pyecharts import options as  opts
  207. from pyecharts.charts import Scatter3D
  208. from pyecharts.faker import Faker
  211. line = Line()
  212. line.add_xaxis(place)
  213. line.add_yaxis("房间面积\n(平米)", area)
  214. line.add_yaxis("房价\n(/万元)", avgs)
  215. line.render("price1_line.html")
  218. price=[float(i)/1 for i in price]
  219. # print(price)
  220. # types=list(map(mapfunc,df.house_type.values))
  221. # type = [224, 56, 168, 1680, 1670]
  222. data = []
  223. # print(fav,type)
  224. # for j in range(len(type)):
  225. #     for k in range(len(fav)):
  226. for j in range(100):
  227.     for k in range(100):
  228.         for i in range(500):
  229.             try:
  230.                 data.append([type[j], favourate_avg[k],price[i]])
  231.             except:
  232.                 continue
  233. # print(data)
  234. scatter = (
  235.      Scatter3D(init_opts=opts.InitOpts(width='900px', height='600px'))  # 初始化
  236.           .add("", data,
  237.                grid3d_opts=opts.Grid3DOpts(
  238.                     width=300, depth=300, rotate_speed=300, is_rotate=True,
  239.                ),)
  241.           # 设置全局配置项
  242.           .set_global_opts(
  243.           title_opts=opts.TitleOpts(title="房型——关注度——价格\n三维关系图"),  # 添加标题
  244.           visualmap_opts=opts.VisualMapOpts(
  245.                max_=300,  # 最大值
  246.                pos_top=200,  # visualMap 组件离容器上侧的距离
  247.                range_color=Faker.visual_color  # 颜色映射
  248.           )
  249.      )
  250.           .render("3D散点图.html")
  251. )
  252. print('数据分析和可视化结束，左边点开~')

最后，别忘记了关注公众号~

分享最新算法！

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-
CiQYvfTG-1594979733395)(D:\CSDN\pic\WeChat Image_20200716151357.jpg)]