一、项目目的：

该项目的主要目的是获取香港以下4区所有楼盘的历史成交数据：

二、项目思路：

  1. 根据以上网址获取需要爬取得楼盘的id号（楼盘id）
  2. 结合楼盘网址，获取该楼盘每个房间的id号（房间id）
  3. 根据楼盘id和房间id，构造新的URL地址，发送Post请求，获取Ajax提供的原代码数据
  4. 通过BeautifulSoup、正则表达式，将获取的html原代码数据解析为关系型数据
  5. 最后，构造二维list将所有结果输出成CSV文件

三、项目原代码：

  1. # -*- coding: utf-8 -*-
  2. # @Time    : 2/23/19 4:35 PM
  3. # @Author  : Wai Mengxin
  4. # @Email   : weimengxin2012@hotmail.com
  5. # @File    : HK_xinjiedong.py
  6. # @Software: PyCharm
  8. import requests
  9. from bs4 import BeautifulSoup
  10. import re
  11. import time
  12. import random
  15. def get_building_code(url):
  16.     '''获得楼层code'''
  17.     res = []
  18.     head = {
  19.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
  20.         'Connection': 'close'}
  21.     s = requests.Session()
  22.     s.keep_alive = False
  23.     html = s.get(url, headers=head).text
  24.     bs = BeautifulSoup(html, 'lxml')
  25.     ob = bs.find("div", id="unitTran-left").find_all("a", href=True)
  26.     for k in ob:
  27.         code = k.attrs["href"]
  28.         res.append(code[-10:])
  29.     s.close()
  30.     return res
  33. def get_house_id(building_acode):
  34.     '''获得房屋的ID'''
  35.     house_id = []
  36.     res = []
  37.     url = 'http://www1.centadata.com/tfs_centadata/Pih2Sln/TransactionHistory.aspx?type=1&code=' + building_acode
  38.     print(url)
  39.     head = {
  40.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
  41.         'Connection': 'close'}
  42.     s = requests.Session()
  43.     s.keep_alive = False
  44.     html = s.get(url, headers=head).text
  45.     bs4 = BeautifulSoup(html, 'lxml')
  46.     ob = bs4.find_all("table", class_="unitTran-sub-table")
  47.     for i in ob:
  48.         house_num = i.find_all("tr", class_="trHasTrans")
  49.         house_id.extend(house_num)
  50.     for it in house_id:
  51.         res.append(it.attrs["id"])
  52.     s.close()
  53.     return res
  56. def get_history_tran(house_id, building_code):
  57.     '''主程序：获得交易历史'''
  58.     url = "http://www1.centadata.com/tfs_centadata/Pih2Sln/Ajax/AjaxServices.asmx/GenTransactionHistoryPinfo"
  59.     headers = [{'User-Agent': "Mozilla/5.0 (X11; CrOS x86_64 10066.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
  60.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
  61.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  62.                 'Connection': 'close'},
  63.                {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/69.0.3497.100 Mobile/13B143 Safari/601.1.46",
  64.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
  65.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  66.                 'Connection': 'close'},
  67.                {'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
  68.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
  69.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  70.                 'Connection': 'close'},
  71.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
  72.                 'Accept': 'application/json, text/plain, */*',
  73.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  74.                 'Connection': 'close'},
  75.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
  76.                 'Accept': 'application/json, text/plain, */*',
  77.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  78.                 'Connection': 'close'}]
  80.     info = {
  81.             "acode": house_id,
  82.             "cblgcode": building_code,
  83.             "cci_price": 4,
  84.             "cultureInfo": "SimplifiedChinese"
  85.             }
  87.     s = requests.Session()
  88.     s.keep_alive = False
  89.     try:
  90.         html = s.post(url, headers=random.choice(headers), data=info, allow_redirects=False)
  91.         time.sleep(0.15)
  92.         # bs = BeautifulSoup(html.content, "lxml")
  93.         res = re.findall(r'\btd>.*?<\b', html.text)
  94.         time.sleep(0.1)
  95.         out_list = []
  96.         finals = []
  97.         need = []
  98.         for i in res:
  99.             result = re.sub("td>|</td><| </td></tr><", "", i)
  100.             out_list.append(result)
  101.         for k in out_list:
  102.             ret = re.sub("</tr></tr></table>|</tr></table></div><|/tr></table><|/tr><|<", "", k)
  103.             finals.append(ret)
  104.         for ite in finals:
  105.             if ite != "":
  106.                 need.append(ite)
  107.     except requests.exceptions.ConnectionError:
  108.         print("ConnectionError....")
  109.         time.sleep(21)
  110.         return get_history_tran(house_id, building_code)
  111.     except requests.exceptions.ChunkedEncodingError:
  112.         print("ProxyError!")
  113.         return get_history_tran(house_id, building_code)
  114.     except TimeoutError:
  115.         print("TimeoutError!")
  116.         time.sleep(5)
  117.         return get_history_tran(house_id, building_code)
  118.     s.close()
  119.     return need
  122. def data_processing(obj):
  123.     '''整理数据'''
  124.     final = []
  125.     res = []
  126.     lis = []
  127.     for k in obj:
  128.         if len(k) <= 2 or k[-2] != "实":
  129.             lis.append(k)
  130.     # try:
  131.     if lis[-1][0] != "@":
  132.         lis.append("----")
  133.     else:
  134.         pass
  136.     start = 5
  137.     t = 0
  138.     while start < len(lis):
  139.         if lis[start][0] != "@" and lis[start] != "----":
  140.             lis.insert(start, "----")
  141.             t += 1
  142.         else:
  143.             pass
  144.         start += 3
  145.         # print(lis)
  147.     # except IndexError:
  148.     #     print("IndexError")
  149.     #     return data_processing(obj)
  151.     num = (len(lis) - 3) / 3
  152.     index = 0
  153.     while index < num:
  154.         res.extend(lis[:3])
  155.         res.extend(lis[3 + 3*index:6 + 3*index])
  156.         li = res.copy()
  157.         final.append(li)
  158.         res.clear()
  159.         index += 1
  160.     return final
  163. def run(urlss):
  164.     result_1 = []
  165.     building_code = get_building_code(urlss)  # 获取楼层code
  167.     for build in building_code:
  168.         house_id = get_house_id(build)   # 获取特定楼层的house id
  169.         for h in house_id:
  170.             raw_data = get_history_tran(h, build)
  171.             # print(raw_data)
  172.             # try:
  173.             data = data_processing(raw_data)
  174.             for item in data:
  175.                 print(item)
  176.             result_1.extend(data)
  177.     return result_1
  180. def output_data(all_data, output_file_name):
  181.     '''该函数将最终数据输出为CSV文件'''
  182.     import pandas as pd
  183.     # name = ["building_name", "net_area", "floor_area", "time", "price", "area_price"]
  184.     table = pd.DataFrame(data=all_data)
  185.     # table = pd.DataFrame(data=all_data)
  186.     table.to_csv("/usr/local/python_files" + output_file_name + ".csv")
  187.     return table
  190. def get_obj_set(url):
  191.     import requests
  192.     from bs4 import BeautifulSoup
  193.     sets = []
  194.     # url = "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=101&info=&code2=&page=0"
  195.     header = {'User-Agent': "Mozilla/5.0 (X11; CrOS x86_64 10066.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
  196.               'Connection': 'close'}
  197.     s = requests.Session()
  198.     s.keep_alive = False
  199.     e = s.get(url, headers=header).text
  200.     html = BeautifulSoup(e, 'lxml')
  201.     obj = html.find_all("td", class_="tdscp1tr")
  202.     for i in obj:
  203.         try:
  204.             h = i.find("a")["href"]
  205.             start = h.find("'")
  206.             res = h[start + 1:start+11]
  207.             sets.append(res)
  208.             print(res)
  209.         except TypeError:
  210.             print("TypeError & PASS")
  211.     return sets
  214. except_code_xinjie = ["XSHNIHSXHT", "DBPPWPPJPW", "XSHNIHZXHN", "GYWKPPKYPS", "GYSEBPYEPK",
  215.                "BYSPWPDOPA", "DBSPWWPXWD", "DFDJURSYRV", "GEPPWPPJPY", "GYGGGPGXPB",
  216.                "GYYYPPGOPS", "VJOVQRSERQ", "XSHMTHNOHT", "BDAAGPYXPA", "XSHSTHZHHT",
  217.                "DBPPWPPSPD", "DEPPWPPSPE", "XSSZZHNSHT", "BDBAGPEHPA", "GWPGGPAEPS",
  218.                "VOOFFRFARR", "LIDHTHXXHT", "WBPPWPPEPP", "WDPPWPPEPB", "AEWPPPSOPW",
  219.                "BSPPGPSEPP", "BEPPWPPAPK", "WBPPWPPRPP", "VDORQRVYRU", "NTHHIHZEHH",
  220.                "KKPPWPPEPS", "DMXSZHLXHD", "WBPPWPPHPP", "BSPPWPPJPP", "BEPPWPPHPW",
  221.                "BGSSBPAXPS", "AEPPWPPAPG", "AEBKPPSJPK", "AEPPWPPRPK", "BEPPWWPOWD"]
  225. url_set = ["http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=208&info=&code2=&page=0",
  226.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=208&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  227.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=301&info=&code2=&page=0",
  228.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=301&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  229.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=302&info=&code2=&page=0",
  230.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=302&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  231.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=303&info=&code2=&page=0",
  232.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=303&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  233.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=304&info=&code2=&page=0",
  234.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=304&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  235.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=306&info=&code2=&page=0",
  236.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=306&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  237.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=307&info=&code2=&page=0",
  238.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=307&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  239.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=307&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  240.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=307&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  241.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=308&info=&code2=&page=0",
  242.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=308&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  243.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=308&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  244.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=308&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  245.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=103&info=&code2=&page=0",
  246.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=309&info=&code2=&page=0",
  247.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=&page=0",
  248.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  249.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  250.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  251.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=4",
  252.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=5",
  253.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=6",
  254.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=7",
  255.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=8",
  256.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=9",
  257.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=401&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=10",
  258.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=&page=0",
  259.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  260.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  261.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  262.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=4",
  263.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=5",
  264.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=402&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=6",
  265.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=403&info=&code2=&page=0",
  266.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=404&info=&code2=&page=0",
  267.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=404&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  268.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=404&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  269.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=404&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  270.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=404&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=4",
  271.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=405&info=&code2=&page=0",
  272.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=405&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  273.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=405&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  274.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=405&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=3",
  275.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=406&info=&code2=&page=0",
  276.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=406&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  277.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=407&info=&code2=&page=0",
  278.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=408&info=&code2=&page=0",
  279.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=408&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  280.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=409&info=&code2=&page=0",
  281.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=409&info=&code2=lord:~lordtype:~tabIdx:0&page=1",
  282.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=409&info=&code2=lord:namec~lordtype:desc~tabIdx:0&page=2",
  283.            "http://www1.centadata.com/paddresssearch1.aspx?type=district17&code=410&info=&code2=&page=0"]
  286. build_code = []
  287. for i in url_set:
  288.     res = get_obj_set(i)
  289.     time.sleep(0.3)
  290.     build_code.extend(res)
  293. final_0 = []
  294. for i in build_code[145:]:
  295.     if i not in except_code_xinjie:
  296.         print(i)
  297.         try:
  298.             urls = "http://www1.centadata.com/tfs_centadata/Pih2Sln/TransactionHistory.aspx?type=3&code=" + i + "&info=basicinfo"
  299.             # urls = "http://hk.centadata.com/transactionhistory.aspx?type=1&code=" + i + "&ci=zh-hk"
  300.             res_1 = run(urls)
  301.             final_0.append(res_1)
  302.         except AttributeError:
  303.             print("AttributeError!")
  304.             final_0.append(i)
  307. def read_csv(name):
  308.     '''该函数读取CSV文件数据'''
  309.     import csv
  310.     csv_file = csv.reader(open("/Users/viemaxwei/Downloads/" + name + ".csv", "r", encoding="utf8"))
  311.     object_website = []
  312.     for i in csv_file:
  313.         object_website.append(i)
  314.         # print(i)
  315.     return object_website
  318. data = []
  319. for i in final_600:
  320.     if isinstance(i, list) is True:
  321.         data.extend(i)
  324. def output_data(all_data, output_file_name):
  325.     '''该函数将最终数据输出为CSV文件'''
  326.     import pandas as pd
  327.     table = pd.DataFrame(data=all_data)
  328.     # table = pd.DataFrame(data=all_data)
  329.     table.to_csv("/Users/viemaxwei/Downloads/" + output_file_name + ".csv")
  330.     return table
  332. results = output_data(data, "港岛")  # 输出最终结果