交接文档中信建投证券智能搜索项目部署文档V7.pdf

fast-spider <- crawled-html <- parse-html <- error-send

登录

  1. sudo ssh -i keyanpingtai.pem ubuntu@82.157.52.202
  2. sudo ssh -i keyanpingtai.pem ubuntu@82.157.46.15

安装mysql

  1. - use mysql;
  2. - alter user 'root'@'%' identified with mysql_native_password by 'T94ESWos0ZPp&xiw';
  3. - docker run -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=T94ESWos0ZPp&xiw --name my_mysql mysql:5.7
  4. - sudo docker run -d -p 3306:3306 -v /home/mysql/conf:/etc/mysql/conf.d -v /home/mysql/data:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=123456 --name mysql01 mysql:5.7

安装postgre

  1. - docker run --name postgres -e POSTGRES_PASSWORD=YLeSqos0Znp -e POSTGRES_USER=root -e POSTGRES_DB=data-platform -p 5432:5432 -d eeb5ef226f19

安装docker-compose

  1. - cd /usr/local/bin
  2. - sudo wget -c  -m kEpnp http://example.org https://github.com/docker/compose/releases/download/1.14.0-rc2/docker-compose-Linux-x86_64
  3. - mv docker-compose-Linux-x86_64 docker-compose
  4. - sudo chmod +x /usr/local/bin/docker-compose

密码

  1. - MySQL 密码：T94ESWos0ZPp&xiw
  2. - postgres 密码：YLeSqos0Znp

DockerCompose

  1. version: '3.2'
  2. services:
  3.   spider-management-web:
  4.     image: harbor.shannonai.com/internal-dp/spider-management-web:v1.2.0
  5.     container_name: spider-management-web
  6.     restart: always
  7.     labels:
  8.       - "traefik.enable=true"
  9.       - "traefik.frontend.rule=PathPrefix:/api"
  10.     ports:
  11.       - "8080"
  12.     volumes:
  13.       - ./internal-dp:/internal-dp
  14.     depends_on:
  15.       - "traefik"
  16.   spider-management-web-frontend:
  17.     image: harbor.shannonai.com/frontend/policy-brain-spider-manage-frontend:v1.2.0
  18.     container_name: spider-management-web-frontend
  19.     restart: always
  20.     labels:
  21.       - "traefik.enable=true"
  22.       - "traefik.frontend.rule=PathPrefix:/"
  23.     ports:
  24.       - "8080"
  25.     volumes:
  26.       - ./config/static-webserver.yaml:/home/work/conf/static-webserver.yaml
  27.       - ./assets:/home/work/assets
  28.     depends_on:
  29.       - "traefik"
  30.   traefik:
  31.     image: traefik:v1.7
  32.     command:
  33.       - "--api --docker --docker.exposedbydefault=false" # Enables the web UI and tells Traefik to listen to docker
  34.       - 
  35.     ports:
  36.       - "80:80"
  37.       - "8003:8080"
  38.     volumes:
  39.       - /var/run/docker.sock:/var/run/docker.sock

  • 换nginx 2

    1. version: '3.2'
    2. services:
    3. spider-management-web:
    4.   image: harbor.shannonai.com/internal-dp/spider-management-web:v1.2.0
    5.   container_name: spider-management-web
    6.   restart: always
    7.   ports:
    8.     - "8080"
    9.   volumes:
    10.     - ./backend/internal-dp:/internal-dp
    11. spider-management-web-frontend:
    12.   image: harbor.shannonai.com/frontend/policy-brain-spider-manage-frontend:v1.2.0
    13.   container_name: spider-management-web-frontend
    14.   restart: always
    15.   ports:
    16.     - "8000:8080"
    17.   volumes:
    18.     - ./frontend/config/static-webserver.yaml:/home/work/conf/static-webserver.yaml
    19.     - ./frontend/assets:/home/work/assets
    20.   depends_on:
    21.     - "spider-management-web"
    22. nginx:
    23.   image: library/nginx:latest
    24.   container_name: nginx
    25.   ports:
    26.     - "80:80"
    27.     - "443:443"
    28.   volumes:
    29.     - ./nginx:/etc/nginx/conf.d
    30.     - ./cert:/etc/nginx/cert
    31.   depends_on:
    32.     - "spider-management-web-frontend"
    33. spider-parse-html-sync:
    34.   image: spider-parse-html-sync:v1.0.0
    35.   container_name: spider-parse-html-sync
    36.   ports:
    37.     - "8001"
    38.   volumes:
    39.     - ./backend/sync-parse-html/main.py:/home/work/main.py
    40.     - ./backend/sync-parse-html/logs:/home/work/logs
    41.     - ./backend/sync-parse-html/database:/home/work/database
    42.   depends_on:
    43.     - "spider-management-web"

    nginx.conf

    server {
  listen       80;
  listen       443 ssl;
  server_name  crawler-common.uibe.info;
  ssl_certificate /etc/nginx/cert/crawler-common.uibe.info.pem;
  ssl_certificate_key /etc/nginx/cert/crawler-common.uibe.info.key;
  ssl_session_timeout 5m;
  ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
  ssl_prefer_server_ciphers on;
  #access_log  /var/log/nginx/host.access.log  main;

  root         /home/work/assets;
  location /api/v1 {
      proxy_pass         http://82.157.52.202:8080;
      # index  index.html index.htm;
  }

  #error_page  404              /404.html;

  # redirect server error pages to the static page /50x.html
  #
  #error_page   500 502 503 504  /50x.html;
  #location = /50x.html {
  #    root   /usr/share/nginx/html;
  #}

  # proxy the PHP scripts to Apache listening on 127.0.0.1:80
  #
  #location ~ \.php$ {
  #    proxy_pass   http://127.0.0.1;
  #}

  # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
  #
  #location ~ \.php$ {
  #    root           html;
  #    fastcgi_pass   127.0.0.1:9000;
  #    fastcgi_index  index.php;
  #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
  #    include        fastcgi_params;
  #}

  # deny access to .htaccess files, if Apache's document root
  # concurs with nginx's one
  #
  #location ~ /\.ht {
  #    deny  all;
  #}
}