【20181010】Python urlparse md5hash

背景

• 在做URL日志与算法之前通常需要对URL进行特征化和归一化，本文介绍部分URLparse相关的库和方法Demo。

urlparse解析url的query并构建字典

import urlparse
url = "http://www.example.org/b.html?a=1&b=2#abc=a"
print(urlparse.urlsplit(url))
print(urlparse.parse_qs(urlparse.urlsplit(url).query))
print(dict(urlparse.parse_qsl(urlparse.urlsplit(url).query)))
===
SplitResult(scheme='http', netloc='www.example.org', path='/b.html', query='a=1&b=2', fragment='abc=a')
{'a': ['1'], 'b': ['2']}
{'a': '1', 'b': '2'}

注意：

1. 在Python3中， urlparse已经被移动到urllib.parse中。

2. 在urlparse中有两个函数：urlparse.parse_qs()和urlparse.parse_qsl()。这两个函数都能解析url中的query字段。如果url的query中有同一个key对应多个value，其中urlparse.parse_qs()可以把该相同key的value放在一个list中。

3. 有时间测试一下，如果url的query中有同一个key对应多个value，那么服务端要怎样接收。

url解码

>>> import urlparse
>>> from urlparse import unquote
>>> url = "http://www.google.com/support/contact/bin/request.py?entity=%7B%22author%22:%22AIe9_BEW4fia2hKVVTrlUwNzhLS-jMdh3isj0rMd7_Cw85R1-YlRNFkUwoDyhH4aMj7AdHsW5A1po8BinbxspAuLBdB-or_3YzCMNXZKYrb50MIIJCZEpb4%22,%22groups%22:%5B%22general%22,%2254296%7C700726330%22%5D,%22trustedMerchantId%22:%22MID_54316%22%7D&client=242&contact_type=anno&hl=en_US"
>>> a = urlparse.urlparse(url).query
>>> b = unquote(a)
>>> b
'entity={"author":"AIe9_BEW4fia2hKVVTrlUwNzhLS-jMdh3isj0rMd7_Cw85R1-YlRNFkUwoDyhH4aMj7AdHsW5A1po8BinbxspAuLBdB-or_3YzCMNXZKYrb50MIIJCZEpb4","groups":["general","54296|700726330"],"trustedMerchantId":"MID_54316"}&client=242&contact_type=anno&hl=en_US'
>>> import HTMLParser
>>> html_parser = HTMLParser.HTMLParser()
>>> txt = html_parser.unescape(b)
>>> txt
u'entity={"author":"AIe9_BEW4fia2hKVVTrlUwNzhLS-jMdh3isj0rMd7_Cw85R1-YlRNFkUwoDyhH4aMj7AdHsW5A1po8BinbxspAuLBdB-or_3YzCMNXZKYrb50MIIJCZEpb4","groups":["general","54296|700726330"],"trustedMerchantId":"MID_54316"}&client=242&contact_type=anno&hl=en_US'
>>> c = urlparse.parse_qsl(txt, True)
>>> c   # c是一个list
[(u'entity', u'{"author":"AIe9_BEW4fia2hKVVTrlUwNzhLS-jMdh3isj0rMd7_Cw85R1-YlRNFkUwoDyhH4aMj7AdHsW5A1po8BinbxspAuLBdB-or_3YzCMNXZKYrb50MIIJCZEpb4","groups":["general","54296|700726330"],"trustedMerchantId":"MID_54316"}'), (u'client', u'242'), (u'contact_type', u'anno'), (u'hl', u'en_US')]
>>> import json
>>> c = dict(c)
>>> d = json.loads(c['entity'])
>>> d
{u'trustedMerchantId': u'MID_54316', u'groups': [u'general', u'54296|700726330'], u'author': u'AIe9_BEW4fia2hKVVTrlUwNzhLS-jMdh3isj0rMd7_Cw85R1-YlRNFkUwoDyhH4aMj7AdHsW5A1po8BinbxspAuLBdB-or_3YzCMNXZKYrb50MIIJCZEpb4'}
>>> print d['groups'][-1]
54296|700726330

计算Hash

from hashlib import md5
url = "http://wxapp.1688.com/wx/offer/576822531818?app_id=e88kTif9Bs&_app_id=e88kTif9Bs&session_key="
print(md5(url).hexdigest())