11_nn.Non-linear Activations

以ReLu举例,ReLU是NLP常用的激活函数

https://pytorch.org/docs/stable/generated/torch.nn.ReLU.html#torch.nn.ReLU

  1. torch.nn.ReLU(inplace=False)
  2. Parameters:
  3.     inplace – can optionally do the operation in-place. Default: False
  4.     #及是否原地替换
  5. Shape:
  6.     Input: (*) where * means any number of dimensions.
  8.     Output: (*) ,same shape as the input.

inplace理解

  1. import torch
  2. from torch import nn
  3. from torch.nn import ReLU
  5. input = torch.tensor([[1, -0.5],
  6.                       [-2, 3]])
  8. class DEMO(nn.Module):
  9.     def __init__(self):
  10.         super(DEMO, self).__init__()
  11.         self.relu1 = ReLU()
  13.     def forward(self,x):
  14.         x = self.relu1(x)
  15.         return x
  17. demo = DEMO()
  19. output = demo(input)
  20. print(output)
  1. #以CIFAR10数据集为例
  2. import torch
  3. import torchvision
  4. from torch import nn
  5. from torch.nn import ReLU, Sigmoid
  6. from torch.utils.data import DataLoader
  7. from torch.utils.tensorboard import SummaryWriter
  9. dataset = torchvision.datasets.CIFAR10('./dataset',train=False,transform=torchvision.transforms.ToTensor(),
  10.                                        download=True)
  11. dataloader = DataLoader(dataset,batch_size=64)
  13. class DEMO(nn.Module):
  14.     def __init__(self):
  15.         super(DEMO, self).__init__()
  16.         self.relu1 = ReLU()
  17.         self.sigmoid = Sigmoid()
  19.     def forward(self,x):
  20.         x = self.sigmoid(x)
  21.         return x
  23. demo = DEMO()
  25. step = 0
  27. writer = SummaryWriter('./logs')
  29. for data in dataloader:
  30.     imgs,targets = data
  31.     output = demo(imgs)
  32.     writer.add_images('input_sigmoid', imgs, step)
  33.     writer.add_images('output_sigmoid', output, step)
  35. writer.close()