53.pdf

    1. # -*- coding: utf-8 -*-
    2. """lstm
    4. Automatically generated by Colaboratory.
    6. Original file is located at
    7.     https://colab.research.google.com/drive/1GX0Rqur8T45MSYhLU9MYWAbycfLH4-Fu
    8. """
    10. !pip install torch
    11. !pip install torchtext
    12. !python -m spacy download en
    15. # K80 gpu for 12 hours
    16. import torch
    17. from torch import nn, optim
    18. from torchtext import data, datasets
    20. print('GPU:', torch.cuda.is_available())
    22. torch.manual_seed(123)
    24. TEXT = data.Field(tokenize='spacy')
    25. LABEL = data.LabelField(dtype=torch.float)
    26. train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)
    28. print('len of train data:', len(train_data))
    29. print('len of test data:', len(test_data))
    31. print(train_data.examples[15].text)
    32. print(train_data.examples[15].label)
    34. # word2vec, glove
    35. TEXT.build_vocab(train_data, max_size=10000, vectors='glove.6B.100d')
    36. LABEL.build_vocab(train_data)
    39. batchsz = 30
    40. device = torch.device('cuda')
    41. train_iterator, test_iterator = data.BucketIterator.splits(
    42.     (train_data, test_data),
    43.     batch_size = batchsz,
    44.     device=device
    45. )
    47. class RNN(nn.Module):
    49.     def __init__(self, vocab_size, embedding_dim, hidden_dim):
    50.         """
    51.         """
    52.         super(RNN, self).__init__()
    54.         # [0-10001] => [100]
    55.         self.embedding = nn.Embedding(vocab_size, embedding_dim)
    56.         # [100] => [256]
    57.         self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=2, 
    58.                            bidirectional=True, dropout=0.5)
    59.         # [256*2] => [1]
    60.         self.fc = nn.Linear(hidden_dim*2, 1)
    61.         self.dropout = nn.Dropout(0.5)
    64.     def forward(self, x):
    65.         """
    66.         x: [seq_len, b] vs [b, 3, 28, 28]
    67.         """
    68.         # [seq, b, 1] => [seq, b, 100]
    69.         embedding = self.dropout(self.embedding(x))
    71.         # output: [seq, b, hid_dim*2]
    72.         # hidden/h: [num_layers*2, b, hid_dim]
    73.         # cell/c: [num_layers*2, b, hid_di]
    74.         output, (hidden, cell) = self.rnn(embedding)
    76.         # [num_layers*2, b, hid_dim] => 2 of [b, hid_dim] => [b, hid_dim*2]
    77.         hidden = torch.cat([hidden[-2], hidden[-1]], dim=1)
    79.         # [b, hid_dim*2] => [b, 1]
    80.         hidden = self.dropout(hidden)
    81.         out = self.fc(hidden)
    83.         return out
    85. rnn = RNN(len(TEXT.vocab), 100, 256)
    87. pretrained_embedding = TEXT.vocab.vectors
    88. print('pretrained_embedding:', pretrained_embedding.shape)
    89. rnn.embedding.weight.data.copy_(pretrained_embedding)
    90. print('embedding layer inited.')
    92. optimizer = optim.Adam(rnn.parameters(), lr=1e-3)
    93. criteon = nn.BCEWithLogitsLoss().to(device)
    94. rnn.to(device)
    96. import numpy as np
    98. def binary_acc(preds, y):
    99.     """
    100.     get accuracy
    101.     """
    102.     preds = torch.round(torch.sigmoid(preds))
    103.     correct = torch.eq(preds, y).float()
    104.     acc = correct.sum() / len(correct)
    105.     return acc
    107. def train(rnn, iterator, optimizer, criteon):
    109.     avg_acc = []
    110.     rnn.train()
    112.     for i, batch in enumerate(iterator):
    114.         # [seq, b] => [b, 1] => [b]
    115.         pred = rnn(batch.text).squeeze(1)
    116.         # 
    117.         loss = criteon(pred, batch.label)
    118.         acc = binary_acc(pred, batch.label).item()
    119.         avg_acc.append(acc)
    121.         optimizer.zero_grad()
    122.         loss.backward()
    123.         optimizer.step()
    125.         if i%10 == 0:
    126.             print(i, acc)
    128.     avg_acc = np.array(avg_acc).mean()
    129.     print('avg acc:', avg_acc)
    132. def eval(rnn, iterator, criteon):
    134.     avg_acc = []
    136.     rnn.eval()
    138.     with torch.no_grad():
    139.         for batch in iterator:
    141.             # [b, 1] => [b]
    142.             pred = rnn(batch.text).squeeze(1)
    144.             #
    145.             loss = criteon(pred, batch.label)
    147.             acc = binary_acc(pred, batch.label).item()
    148.             avg_acc.append(acc)
    150.     avg_acc = np.array(avg_acc).mean()
    152.     print('>>test:', avg_acc)
    154. for epoch in range(10):
    156.     eval(rnn, test_iterator, criteon)
    157.     train(rnn, train_iterator, optimizer, criteon)

    lstm.ipynb