内容可见注释

    1. #simple note in learning pytorch
    3. import torch
    4. import numpy
    6. '''
    7. # create tensor
    8. x = torch.empty(5,3)
    9. y = torch.rand(5,3)
    10. z = torch.zeros(5,3)
    11. print(y)
    12. print(x)
    13. print(z)
    14. print (x.dtype)
    15. print(z.dtype)
    17. a=torch.tensor([5,5,3])
    18. print(a)
    20. b=torch.tensor([6,6,8])
    21. print(b)
    23. #basic operation
    24. b=a+b
    25. print(b)
    26. b=torch.add(a,b)
    27. print(b)
    28. #special in place
    29. b.add_(a)
    30. print(b)
    32. #resize
    33. test=torch.rand(4,2)
    34. print(test)
    35. test=test.view(8)
    36. print(test)
    38. test=torch.rand(1)
    39. print(test)
    40. test2=test.item()
    41. print(test2)
    44. # transition between torch and numpy
    45. # they share the data
    46. a=torch.ones(5)
    47. print (a)
    48. b=a.numpy()
    49. print(b)
    50. b[1]=2
    51. print(a)
    52. print(b)
    54. a =numpy.ones(5)
    55. b=torch.from_numpy(a)
    56. print(a)
    57. print(b)
    59. # if you have a GPU you can .....
    60. if torch.cuda.is_available():
    61.     device=torch.device("cuda")
    62.     print("yes I have")
    63.     y=torch.ones_like(b,device=device)
    64.     b=b.to(device)
    68. #warming up
    69. # try to build a simple two layer neural net (using numpy)
    70. N,D_in,H,D_out = 64,1000,100,10
    71. x=numpy.random.randn(N,D_in)
    72. y=numpy.random.randn(N,D_out)
    74. w1=numpy.random.randn(D_in,H)
    75. w2=numpy.random.randn(H,D_out)
    77. learning_rate=1e-6
    78. for t in range(500):
    79.     # forward pass
    80.     h=x.dot(w1)
    81.     h_Relu=numpy.maximum(h,0)
    82.     y_pred=h_Relu.dot(w2)
    84.     #compute loss and use square loss
    85.     loss = numpy.square(y_pred-y).sum()
    86.     print(t,loss)
    88.     #backward pass
    89.     # 1. compute gradiend
    90.     grad_y_pred=2.0*(y_pred-y)
    91.     grad_w2 =h_Relu.T.dot(grad_y_pred)
    92.     grad_h_relu=grad_y_pred.dot(w2.T)
    93.     grad_h = grad_h_relu.copy()
    94.     grad_h[h<0]=0
    95.     grad_w1 = x.T.dot(grad_h)
    97.     # Update weights
    98.     w1 -= learning_rate * grad_w1
    99.     w2 -= learning_rate * grad_w2
    101. #now try to use torch to do the job
    103. N,D_in,H,D_out = 64,1000,100,10
    104. x=torch.randn(N,D_in)
    105. y=torch.randn(N,D_out)
    107. w1=torch.randn(D_in,H)
    108. w2=torch.randn(H,D_out)
    110. learning_rate=1e-6
    111. for t in range(500):
    112.     # forward pass
    113.     h=x.mm(w1)
    114.     h_Relu=h.clamp(min=0)
    115.     y_pred=h_Relu.mm(w2)
    117.     #compute loss and use square loss
    118.     loss = (y_pred-y).pow(2).sum().item()
    119.     print(t,loss)
    121.     #backward pass
    122.     # 1. compute gradiend
    123.     grad_y_pred=2.0*(y_pred-y)
    124.     grad_w2 =h_Relu.t().mm(grad_y_pred)
    125.     grad_h_relu=grad_y_pred.mm(w2.t())
    126.     grad_h = grad_h_relu.clone()
    127.     grad_h[h<0]=0
    128.     grad_w1 = x.T.mm(grad_h)
    130.     # Update weights
    131.     w1 -= learning_rate * grad_w1
    132.     w2 -= learning_rate * grad_w2
    135. #final version by using auto_backward
    137. N,D_in,H,D_out = 64,1000,100,10
    139. x=torch.randn(N,D_in,requires_grad=True)#default is False
    140. y=torch.randn(N,D_out,requires_grad=True)
    142. w1=torch.randn(D_in,H,requires_grad=True)
    143. w2=torch.randn(H,D_out,requires_grad=True)
    145. learning_rate=1e-6
    146. for t in range(500):
    147.     # forward pass
    149.     y_pred =x.mm(w1).clamp(min=0).mm(w2)
    150.     #compute loss and use square loss
    151.     loss = (y_pred-y).pow(2).sum()
    152.     #
    153.     loss_num=loss.item()
    154.     print(t, loss_num)
    155.     #backward pass
    156.     # compute gradiend
    159.     loss.backward()
    161.     # Update weights
    162.     with torch.no_grad():
    163.         w1 -= learning_rate * w1.grad
    164.         w2 -= learning_rate * w2.grad
    165.         w1.grad.zero_()
    166.         w2.grad.zero_()
    167. '''
    170. import torch.nn as nn
    171. import time
    172. time_start=time.time()
    173. N,D_in,H,D_out = 64,1000,100,10
    175. x=torch.randn(N,D_in,requires_grad=True)#default is False
    176. y=torch.randn(N,D_out,requires_grad=True)
    178. model=torch.nn.Sequential(
    179.     torch.nn.Linear(D_in,H),
    180.     torch.nn.ReLU(),
    181.     torch.nn.Linear(H,D_out),
    182. )
    184. if torch.cuda.is_available():
    185.     model=model.cuda()
    186.     x=x.cuda()
    187.     y=y.cuda()
    189. loss_func=nn.MSELoss(reduction='sum')
    191. learning_rate=1e-6
    192. for t in range(20000):
    193.     # forward pass
    195.     y_pred =model(x)
    197.     #compute loss and use square loss
    198.     loss = loss_func(y_pred , y)
    200.     #loss_num=loss.item()
    201.     print(t)
    202.     #backward pass
    203.     # compute gradiend
    206.     loss.backward()
    207.     with torch.no_grad():
    208.         for param in model.parameters():
    209.             param-=learning_rate*param.grad
    210.         model.zero_grad()
    211.     # Update weights
    212. time_end=time.time()
    213. print('totally cost',time_end-time_start)

    最后分别使用GPU和CPU运行尝试
    image.png

    image.png

    从而有了若干问题

    1. 使用CPU运行时间只有GPU一半，，反而更快
    2. 使用了模型进行统一化之后，反而收敛效果更差，loss值降不下来