两个选项卡关闭第一个

  1. package main
  3. import (
  4.     "context"
  5.     "time"
  7.     "github.com/chromedp/cdproto/cdp"
  8.     "github.com/chromedp/cdproto/page"
  9.     "github.com/chromedp/chromedp"
  10. )
  12. func main() {
  13.     // create the first tab
  14.     c, _ := chromedp.NewExecAllocator(context.Background(), chromedp.Flag("headless", false))
  15.     ctx1, _ := chromedp.NewContext(c)
  16.     chromedp.Run(ctx1)
  18.     // make second tab
  19.     ctx2, cancel := chromedp.NewContext(ctx1)
  20.     defer cancel() // close the second tab
  21.     chromedp.Run(ctx2)
  23.     // close the first tab, not closing the browser
  24.     if err := page.Close().Do(cdp.WithExecutor(ctx1, chromedp.FromContext(ctx1).Target)); err != nil {
  25.         panic(err)
  26.     }
  27.     time.Sleep(time.Second * 2)
  29.     // closing the second tab kills the browser
  30. }

多个选项卡

  1. func ExampleNewContext_manyTabs() { 
  2.      // new browser, first tab 
  3.      ctx1, cancel := chromedp.NewContext(context.Background()) 
  4.      defer cancel() 
  6.      // ensure the first tab is created 
  7.      if err := chromedp.Run(ctx1); err != nil { 
  8.          panic(err) 
  9.      } 
  11.      // same browser, second tab 
  12.      ctx2, _ := chromedp.NewContext(ctx1) 
  14.      // ensure the second tab is created 
  15.      if err := chromedp.Run(ctx2); err != nil { 
  16.          panic(err) 
  17.      } 
  19.      c1 := chromedp.FromContext(ctx1) 
  20.      c2 := chromedp.FromContext(ctx2) 
  22.      fmt.Printf("Same browser: %t\n", c1.Browser == c2.Browser) 
  23.      fmt.Printf("Same tab: %t\n", c1.Target == c2.Target) 
  25.      // Output: 
  26.      // Same browser: true 
  27.      // Same tab: false 
  28.  }

仅当存在节点时单击

  1. err := chromedp.Run(ctx,
  2.     chromedp.Navigate("[any_url_here]"),
  3.     chromedp.ActionFunc(func(ctx context.Context) error {
  4.         var nodes []*cdp.Node
  5.         if err := chromedp.Nodes(xpath, &nodes, chromedp.AtLeast(0).Do(ctx); err != nil { return err }
  6.         if len(nodes) == 0 { return nil } // nothing to do
  7.         return chromedp.MouseClickNode(nodes[0]).Do(ctx)
  8.     }),
  9.     chromedp.Evaluate("window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' });", &res),
  10. )

多个urls访问抓取

  1. package main
  3. import (
  4.     "context"
  5.     "log"
  6.     "time"
  8.     "github.com/chromedp/chromedp"
  9. )
  11. func main() {
  12.     options := append(chromedp.DefaultExecAllocatorOptions[:],
  13.         chromedp.DisableGPU,
  14.         chromedp.NoSandbox,
  15.         chromedp.Flag("ignore-certificate-errors", true),
  16.         chromedp.WindowSize(1400, 900),
  17.     )
  19.     aCtx, aCancel := chromedp.NewExecAllocator(context.Background(), options...)
  20.     defer aCancel()
  22.     ctx, cancel := chromedp.NewContext(aCtx)
  23.     defer cancel()
  25.     // a no-op action to allocate the browser
  26.     err := chromedp.Run(ctx, chromedp.ActionFunc(func(_ context.Context) error {
  27.         return nil
  28.     }))
  29.     if err != nil {
  30.         log.Fatalln(err)
  31.     }
  33.     urls := []string{"https://www.google.com", "https://www.bing.com"}
  34.     for _, url := range urls {
  35.         func() {
  36.             xCtx, xCancel := context.WithTimeout(ctx, time.Second*90)
  37.             defer xCancel()
  38.             err := chromedp.Run(xCtx, chromedp.Tasks{
  39.                 chromedp.Navigate(url),
  40.             })
  41.             if err != nil {
  42.                 log.Printf("%s failed: %s ", url, err)
  43.                 return
  44.             }
  45.             log.Println("success: ", url)
  46.         }()
  47.     }
  48. }

chromedp一些options

  1.     opts := append(chromedp.DefaultExecAllocatorOptions[:],
  2.         // 禁用GPU，不显示GUI
  3.         chromedp.DisableGPU,
  4.         // 取消沙盒模式
  5.         chromedp.NoSandbox,
  6.         // 指定浏览器分辨率
  7.         //chromedp.WindowSize(1600, 900),
  8.         // 设置UA，防止有些页面识别headless模式
  9.         chromedp.UserAgent(utils.UserAgent),
  10.         // 隐身模式启动
  11.         chromedp.Flag("incognito", true),
  12.         // 忽略证书错误
  13.         chromedp.Flag("ignore-certificate-errors", true),
  14.         // 窗口最大化
  15.         chromedp.Flag("start-maximized", true),
  16.         // 不加载图片, 提升速度
  17.         chromedp.Flag("disable-images", true),
  18.         chromedp.Flag("blink-settings", "imagesEnabled=false"),
  19.         // 禁用扩展
  20.         chromedp.Flag("disable-extensions", true),
  21.         // 禁止加载所有插件
  22.         chromedp.Flag("disable-plugins", true),
  23.         // 禁用浏览器应用
  24.         chromedp.Flag("disable-software-rasterizer", true),
  25.         //chromedp.Flag("remote-debugging-port","9222"),
  26.         //chromedp.Flag("debuggerAddress","127.0.0.1:9222"),
  27.         chromedp.Flag("user-data-dir", "./.cache"),
  28.         //chromedp.Flag("excludeSwitches", "enable-automation"),
  29.         // 设置用户数据目录
  30.         //chromedp.UserDataDir(dir),
  31.         //chromedp.ExecPath("C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe"),
  32.     )

并行两个协程就出验证码，且继续并行两个协程验证码无效

chromedp提交验证码速度过快也会导致验证码无效

there an easy way to set cookie?

link

Test

case1:

  • one channel
  • 平均每2s一个请求链接