核心概念

Document

文本，python中的字符串

  1. document = "Human machine interface for lab abc computer applications"

Corpus

Document的集合，作为模型的输入

  1. text_corpus = [
  2.     "Human machine interface for lab abc computer applications",
  3.     "A survey of user opinion of computer system response time",
  4.     "The EPS user interface management system",
  5.     "System and human system engineering testing of EPS",
  6.     "Relation of user perceived response time to error measurement",
  7.     "The generation of random binary unordered trees",
  8.     "The intersection graph of paths in trees",
  9.     "Graph minors IV Widths of trees and well quasi ordering",
  10.     "Graph minors A survey",
  11. ]

为了给语料中的每个单词一个ID，可以使用gensim.corpora.Dictionary

  1. # Create a set of frequent words
  2. stoplist = set('for a of the and to in'.split(' '))
  3. # Lowercase each document, split it by white space and filter out stopwords
  4. texts = [[word for word in document.lower().split() if word not in stoplist]
  5.          for document in text_corpus]
  7. # Count word frequencies
  8. from collections import defaultdict
  9. frequency = defaultdict(int)
  10. for text in texts:
  11.     for token in text:
  12.         frequency[token] += 1
  14. # Only keep words that appear more than once
  15. processed_corpus = [[token for token in text if frequency[token] > 1] for text in texts]
  16. pprint.pprint(processed_corpus)
  18. '''
  19. [['human', 'interface', 'computer'],
  20.  ['survey', 'user', 'computer', 'system', 'response', 'time'],
  21.  ['eps', 'user', 'interface', 'system'],
  22.  ['system', 'human', 'system', 'eps'],
  23.  ['user', 'response', 'time'],
  24.  ['trees'],
  25.  ['graph', 'trees'],
  26.  ['graph', 'minors', 'trees'],
  27.  ['graph', 'minors', 'survey']]
  28.  '''
  30. from gensim import corpora
  32. dictionary = corpora.Dictionary(processed_corpus)
  33. print(dictionary)
  35. '''
  36. Dictionary(12 unique tokens: ['computer', 'human', 'interface', 'response', 'survey']...)
  37. '''

Vector

文档的向量表示

  1. pprint.pprint(dictionary.token2id)  # get token2id
  2. """
  3. {'computer': 0,
  4.  'eps': 8,
  5.  'graph': 10,
  6.  'human': 1,
  7.  'interface': 2,
  8.  'minors': 11,
  9.  'response': 3,
  10.  'survey': 4,
  11.  'system': 5,
  12.  'time': 6,
  13.  'trees': 9,
  14.  'user': 7}
  15.  """
  16. new_doc = "Human computer interaction"
  17. new_vec = dictionary.doc2bow(new_doc.lower().split())  # change document to vector
  18. print(new_vec)
  19. """
  20. [(0, 1), (1, 1)]  # first is token id, second is #token, token not in dictionary will be ignored
  21. """
  22. bow_corpus = [dictionary.doc2bow(text) for text in processed_corpus]
  23. pprint.pprint(bow_corpus)
  24. """
  25. [[(0, 1), (1, 1), (2, 1)],
  26.  [(0, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)],
  27.  [(2, 1), (5, 1), (7, 1), (8, 1)],
  28.  [(1, 1), (5, 2), (8, 1)],
  29.  [(3, 1), (6, 1), (7, 1)],
  30.  [(9, 1)],
  31.  [(9, 1), (10, 1)],
  32.  [(9, 1), (10, 1), (11, 1)],
  33.  [(4, 1), (10, 1), (11, 1)]]
  34.  """

Model

进行向量变换的算法，将原始的文档向量空间转换到目标空间

  1. from gensim import models
  3. # train the model
  4. tfidf = models.TfidfModel(bow_corpus)
  6. # transform the "system minors" string
  7. words = "system minors".lower().split()
  8. print(tfidf[dictionary.doc2bow(words)])
  9. """
  10. [(5, 0.5898341626740045), (11, 0.8075244024440723)]
  11. """
  12. from gensim import similarities
  14. index = similarities.SparseMatrixSimilarity(tfidf[bow_corpus], num_features=12)
  15. query_document = 'system engineering'.split()
  16. query_bow = dictionary.doc2bow(query_document)
  17. sims = index[tfidf[query_bow]]
  18. for document_number, score in sorted(enumerate(sims), key=lambda x: x[1], reverse=True):
  19.     print(document_number, score)  # similarity between query and each document in bow_corpus
  20. """
  21. 3 0.7184812
  22. 2 0.41707572
  23. 1 0.32448703
  24. 0 0.0
  25. 4 0.0
  26. 5 0.0
  27. 6 0.0
  28. 7 0.0
  29. 8 0.0
  30. """

小结

documents of corpus -> vector representation -> model transform raw vector

文本->向量

基本方法和上面corpus、vector介绍的一致

流式加载

  1. from smart_open import open  # for transparently opening remote files
  4. class MyCorpus(object):  # 迭代器
  5.     def __iter__(self):
  6.         for line in open('https://radimrehurek.com/gensim/mycorpus.txt'):
  7.             # assume there's one document per line, tokens separated by whitespace
  8.             yield dictionary.doc2bow(line.lower().split())
  10. corpus_memory_friendly = MyCorpus()  # doesn't load the corpus into memory!
  11. for vector in corpus_memory_friendly:  # load one vector into memory at a time
  12.     print(vector)
  13. """
  14. [(0, 1), (1, 1), (2, 1)]
  15. [(0, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)]
  16. [(2, 1), (5, 1), (7, 1), (8, 1)]
  17. [(1, 1), (5, 2), (8, 1)]
  18. [(3, 1), (6, 1), (7, 1)]
  19. [(9, 1)]
  20. [(9, 1), (10, 1)]
  21. [(9, 1), (10, 1), (11, 1)]
  22. [(4, 1), (10, 1), (11, 1)]
  23. """
  1. from six import iteritems
  2. # collect statistics about all tokens
  3. dictionary = corpora.Dictionary(line.lower().split() for line in open('https://radimrehurek.com/gensim/mycorpus.txt'))
  5. # remove stop words and words that appear only once
  6. stop_ids = [
  7.     dictionary.token2id[stopword]
  8.     for stopword in stoplist
  9.     if stopword in dictionary.token2id
  10. ]
  11. once_ids = [tokenid for tokenid, docfreq in iteritems(dictionary.dfs) if docfreq == 1]
  12. dictionary.filter_tokens(stop_ids + once_ids)  # remove stop words and words that appear only once
  13. dictionary.compactify()  # remove gaps in id sequence after words that were removed
  14. print(dictionary)
  15. """
  16. Dictionary(12 unique tokens: ['computer', 'human', 'interface', 'response', 'survey']...)
  17. """

Gensim的一大特色在于对内存的依赖小，支持生成器

相似度查询

  1. from collections import defaultdict
  2. from gensim import corpora
  4. documents = [
  5.     "Human machine interface for lab abc computer applications",
  6.     "A survey of user opinion of computer system response time",
  7.     "The EPS user interface management system",
  8.     "System and human system engineering testing of EPS",
  9.     "Relation of user perceived response time to error measurement",
  10.     "The generation of random binary unordered trees",
  11.     "The intersection graph of paths in trees",
  12.     "Graph minors IV Widths of trees and well quasi ordering",
  13.     "Graph minors A survey",
  14. ]
  16. # remove common words and tokenize
  17. stoplist = set('for a of the and to in'.split())
  18. texts = [
  19.     [word for word in document.lower().split() if word not in stoplist]
  20.     for document in documents
  21. ]
  23. # remove words that appear only once
  24. frequency = defaultdict(int)
  25. for text in texts:
  26.     for token in text:
  27.         frequency[token] += 1
  28. texts = [
  29.     [token for token in text if frequency[token] > 1]
  30.     for text in texts
  31. ]
  32. # build corpus vector
  33. dictionary = corpora.Dictionary(texts)
  34. corpus = [dictionary.doc2bow(text) for text in texts]
  35. # lsi model: (token_id, topic relation)
  36. from gensim import models
  37. lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=2)
  38. doc = "Human computer interaction"
  39. vec_bow = dictionary.doc2bow(doc.lower().split())
  40. vec_lsi = lsi[vec_bow]  # convert the query to LSI space
  41. print(vec_lsi)
  42. """
  43. [(0, 0.4618210045327158), (1, 0.07002766527900064)]
  44. """
  45. # initialize query index
  46. from gensim import similarities
  47. index = similarities.Similarity(lsi[corpus])  # transform corpus to LSI space and index it
  48. # save and load index
  49. index.save('/tmp/deerwester.index')
  50. index = similarities.MatrixSimilarity.load('/tmp/deerwester.index')
  51. # query similarity
  52. sims = index[vec_lsi]  # perform a similarity query against the corpus
  53. print(list(enumerate(sims)))  # print (document_number, document_similarity) 2-tuples
  54. """
  55. [(0, 0.998093), (1, 0.93748635), (2, 0.9984453), (3, 0.9865886), (4, 0.90755945), (5, -0.12416792), (6, -0.10639259), (7, -0.09879464), (8, 0.050041765)]
  56. """
  57. sims = sorted(enumerate(sims), key=lambda item: -item[1])
  58. for i, s in enumerate(sims):
  59.     print(s, documents[i])
  60. """
  61. (2, 0.9984453) Human machine interface for lab abc computer applications
  62. (0, 0.998093) A survey of user opinion of computer system response time
  63. (3, 0.9865886) The EPS user interface management system
  64. (1, 0.93748635) System and human system engineering testing of EPS
  65. (4, 0.90755945) Relation of user perceived response time to error measurement
  66. (8, 0.050041765) The generation of random binary unordered trees
  67. (7, -0.09879464) The intersection graph of paths in trees
  68. (6, -0.10639259) Graph minors IV Widths of trees and well quasi ordering
  69. (5, -0.12416792) Graph minors A survey
  70. """

Word2Vec

加载预训练模型

  1. import gensim.downloader as api
  2. wv = api.load('word2vec-google-news-300')  # load pretrained model
  4. for i, word in enumerate(wv.vocab):  # vocab: [word]
  5.     if i == 10:
  6.         break
  7.     print(word)
  8. """
  10. </s>
  11. in
  12. for
  13. that
  14. is
  15. on
  16. ##
  17. The
  18. with
  19. said
  20. """
  22. try:
  23.     vec_cameroon = wv['cameroon']
  24. except KeyError:
  25.     print("The word 'cameroon' does not appear in this model")
  26. """
  27. The word 'cameroon' does not appear in this model
  28. """
  30. pairs = [
  31.     ('car', 'minivan'),   # a minivan is a kind of car
  32.     ('car', 'bicycle'),   # still a wheeled vehicle
  33.     ('car', 'airplane'),  # ok, no wheels, but still a vehicle
  34.     ('car', 'cereal'),    # ... and so on
  35.     ('car', 'communism'),
  36. ]
  37. for w1, w2 in pairs:
  38.     print('%r\t%r\t%.2f' % (w1, w2, wv.similarity(w1, w2)))  # similarity: compute two word cosine similarity
  39. """
  40. 'car'   'minivan'       0.69
  41. 'car'   'bicycle'       0.54
  42. 'car'   'airplane'      0.42
  43. 'car'   'cereal'        0.14
  44. 'car'   'communism'     0.06
  45. """
  47. print(wv.most_similar(positive=['car', 'minivan'], topn=5))
  48. """
  49. [('SUV', 0.853219211101532), ('vehicle', 0.8175784349441528), ('pickup_truck', 0.7763689160346985), ('Jeep', 0.7567334175109863), ('Ford_Explorer', 0.756571888923645)]
  50. """
  52. print(wv.doesnt_match(['fire', 'water', 'land', 'sea', 'air', 'car']))
  53. """
  54. car
  55. """

训练自己的模型

  1. from gensim.test.utils import datapath
  2. from gensim import utils
  3. # 加载数据，可以是迭代器、生成器
  4. class MyCorpus(object):
  5.     """An interator that yields sentences (lists of str)."""
  7.     def __iter__(self):
  8.         corpus_path = datapath('lee_background.cor')
  9.         for line in open(corpus_path):
  10.             # assume there's one document per line, tokens separated by whitespace
  11.             yield utils.simple_preprocess(line)
  14. import gensim.models
  16. sentences = MyCorpus()
  17. model = gensim.models.Word2Vec(sentences=sentences)
  18. # The trained word vectors are stored in a KeyedVectors instance in model.wv:
  19. vector = model.wv['computer']  # numpy vector of a word
  20. # to trim unneeded model state = use much less RAM and allow fast loading and memory sharing (mmap).
  21. word_vectors = model.wv
  22. del model

Word2Vec参数

  • sentences (iterable of iterables, optional) – The sentences iterable can be simply a list of lists of tokens, but for larger corpora, consider an iterable that streams the sentences directly from disk/network.
  • min_count (int, optional) – Ignores all words with total frequency lower than this.
  • size (int, optional) – Dimensionality of the word vectors.
  • window (int, optional) – Maximum distance between the current and predicted word within a sentence.
  • workers (int, optional) – Use these many worker threads to train the model (=faster training with multicore machines). install Cython to make it work!
  • sg ({0, 1}, optional) – Training algorithm: 1 for skip-gram; otherwise CBOW.
  • cbow_mean ({0, 1}, optional) – If 0, use the sum of the context word vectors. If 1, use the mean, only applies when cbow is used.
  • hs ({0, 1}, optional) – If 1, hierarchical softmax will be used for model training. If 0, and negative is non-zero, negative sampling will be used.
  • negative (int, optional) – If > 0, negative sampling will be used, the int for negative specifies how many “noise words” should be drawn (usually between 5-20). If set to 0, no negative sampling is used.
  • sample (float, optional) – The threshold for configuring which higher-frequency words are randomly downsampled, useful range is (0, 1e-5).

  • compute_loss (bool, optional) – If True, computes and stores loss value which can be retrieved using get_latest_training_loss().

    加载与保存

  • 需要继续训练

model.save(filename),model.load(filename)

  • 不需要继续训练 ```python from gensim.test.utils import get_tmpfile from gensim.models import KeyedVectors

path = get_tmpfile(“wordvectors.kv”)

model.wv.save(path) wv = KeyedVectors.load(“model.wv”, mmap=’r’) vector = wv[‘computer’] # numpy vector of a word

Gensim can also load word vectors in the “word2vec C format”, as a KeyedVectors instance:

from gensim.test.utils import datapath wv_from_text = KeyedVectors.load_word2vec_format(datapath(‘word2vec_pre_kv_c’), binary=False) # C text format wv_from_bin = KeyedVectors.load_word2vec_format(datapath(“euclidean_vectors.bin”), binary=True) # C bin format

  2. - 只用于初始化词向量和word2index
  3. ```python
  4. import gensim
  5. import numpy as np
  6. # train word2vec
  7. # self.corpus: [[word]]
  8. model = gensim.models.word2vec.Word2Vec(self.corpus,size=300,min_count=1)
  9. # model.save('word2vec_redial')
  10. # save word embedding in numpy format
  11. word2embedding = [[0] * 300] * 4 + [model[word] for word in word2index]+[[0]*300]
  12. print(np.shape(word2embedding))
  13. np.save('word2vec_redial.npy', word2embedding)
  14. # word2index
  15. word2index = {word: i + 4 for i, word in enumerate(model.wv.index2word)}
  16. word2index['_split_']=len(word2index)+4
  17. json.dump(word2index, open('word2index_redial.json', 'w', encoding='utf-8'), ensure_ascii=False)
  19. # initialize word embedding in model
  20. e = nn.Embedding(len(dictionary)+4, embedding_size, padding_idx)
  21. e.weight.data.copy_(torch.from_numpy(np.load('word2vec_redial.npy')))