提醒本地 s3 在内网中提供 grafana 截图保存

单机

用于测试

docker-compose.yml

version: '3.7'
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
  minio1:
    image: minio/minio
    volumes:
      - data1-1:/data1
      - data1-2:/data2
    expose:
      - "9000"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    command: server http://minio{1...4}/data{1...2}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
  minio2:
    image: minio/minio
    volumes:
      - data2-1:/data1
      - data2-2:/data2
    expose:
      - "9000"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    command: server http://minio{1...4}/data{1...2}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
  minio3:
    image: minio/minio
    volumes:
      - data3-1:/data1
      - data3-2:/data2
    expose:
      - "9000"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    command: server http://minio{1...4}/data{1...2}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
  minio4:
    image: minio/minio
    volumes:
      - data4-1:/data1
      - data4-2:/data2
    expose:
      - "9000"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    command: server http://minio{1...4}/data{1...2}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
  nginx:
    image: nginx:1.19.2-alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9000:9000"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
  data1-1:
  data1-2:
  data2-1:
  data2-2:
  data3-1:
  data3-2:
  data4-1:
  data4-2:

nginx.conf

user  nginx;
worker_processes  auto;
error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
}
http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    #tcp_nopush     on;
    keepalive_timeout  65;
    #gzip  on;
    # include /etc/nginx/conf.d/*.conf;
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
    }
    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;
         # To allow special characters in headers
         ignore_invalid_headers off;
         # Allow any size file to be uploaded.
         # Set to a value such as 1000m; to restrict file size to a specific value
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;
            proxy_pass http://minio;
        }
    }
}

别名

alias ls='mc ls'
alias cp='mc cp'
alias cat='mc cat'
alias mkdir='mc mb'
alias pipe='mc pipe'
alias find='mc find'

运维问题

The difference between the request time and the server’s time is too large？

• 原因：linux服务器时区的问题
• 解决方案：调整系统时间和硬件时间

# 查看系统时间
date
# 查看硬件时间
hwclock
# 安装ntpdate时间同步工具
yum -y install ntp ntpdate
# 设置系统时间与网络时间同步
ntpdate cn.pool.ntp.org
# 将系统时间写入硬件时间
hwclock --systohc

HA

minio

命令行

MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address :8888 http://server{1...3}/home/minio/data{1...4}

开机启动

chmod a+x /etc/rc.d/rc.local
# minio要用绝对路径
echo "MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 /usr/bin/minio server --address :8888 http://server{1...3}/home/minio/data{1...4}  >/dev/null 2>&1 &" >> /etc/rc.d/rc.local

docker swarm

较为繁琐

docker-compose

仅用于单机多节点测试

nginx::minio.conf

upstream minio_http {
    least_conn;
    server 172.19.129.62:8888 max_fails=3 fail_timeout=5s;
    server 172.19.129.68:8888 max_fails=3 fail_timeout=5s;
    server 172.19.129.69:8888 max_fails=3 fail_timeout=5s;
}
server {
    listen 443 ssl;
    server_name abc.do.io;
    ssl_certificate /certs/do.io.cert;
    ssl_certificate_key /certs/do.io.key;
    client_max_body_size 5000m;  # 上传大文件
    ssl_session_timeout  600m;
    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
    location / {
        proxy_pass http://minio_http;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
    }
}

管理

version: '3.7'
services:
  mc:
    image: minio/mc
    container_name: mc
    entrypoint: start.sh
    restart: always
    environment:
      # bucket name
      BUK: haifeng
    volumes:
      # 数据目录
      - ./data:/data
      # 启动脚本
      - ./start.sh:/start.sh:ro
    entrypoint: ["/start.sh"]

start.sh

#!/bin/sh
# 修改http后的IP为nginx代理的ip，修改后面minio的登录帐号密码
mc alias set minio http://$(ping op -c 1 | sed '1{s/[^(]*(//;s/).*//;q}'):9000 minio minio123 --api s3v4
mc policy set public minio/$BUK
# 需创建目录，mc cp没有创建目录功能
mkdir -p /data/$BUK
mc cp -r minio/$BUK /data
mc mirror -w /data/$BUK minio/$BUK

mc指令

# 拉取mc客户端镜像
docker pull minio/mc
# 运行,映射配置目录和数据目录
docker run -itd --name mc -e "BUK=minio/haifeng" -v $PWD/mc_config:/root/.mc/ -v $PWD/data/data --entrypoint=/bin/sh minio/mc
# 添加一个云存储服务
# mc alias set <ALIAS> <YOUR-S3-ENDPOINT> <YOUR-ACCESS-KEY> <YOUR-SECRET-KEY> [--api API-SIGNATURE]
docker exec -it mc bash
]$ mc alias set minio https://abc.do.io minio minio123 --api s3v4
# 设置访问权限
]$ mc policy set public $BUK
# 映射路径 # 先复制文件再同步
]$ mc cp -r $BUK /data
# 必要时可加参数 --force --overwrite
# mc mirror [FLAGS] SOURCE TARGET
]$ nohup mc mirror -w /data $BUK &

• 权限4种：none, download, upload, public
• mc命令方式和Minio浏览器上设置策略二选一

问题

在用minio存储作为rancher的存储时异常，首次运行后无法再次up。

version: '3.7'
services:
  rancher:
    image: rancher/rancher:v2.5.5
    container_name: rancher
    restart: unless-stopped
    privileged: true
    environment:
      - TZ=Asia/Shanghai
      - AUDIT_LEVEL=3
    ports:
      - "8443:443"
    volumes:
      # minio ha store
       - ../minio_mc/data/haifeng/rancher:/var/lib/rancher
       - ../minio_mc/data/haifeng/auditlog:/var/log/auditlog

S3 兼容 aws

• 创建 s3 数据目录

  mkdir s3

• 运行minio 兼容aws s3模式，后台模式，无视对话退出，输出log到/s3

  nohup ./minio --compat server $PWD/s3 >$PWD/s3/minio.log 2>&1 &
  compat是minio兼容aws s3的参数，其他参数是linux执行程序的参数，有疑问请自行查阅

• 实际采用指令

  mkdir ./s3
  MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 ./minio --compat server $PWD/s3 --address :1001 >$PWD/s3/minio.log 2>&1 &

• 开机启动

  chmod a+x /etc/rc.d/rc.local
  # minio要用绝对路径
  echo "MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio --compat server /s3 --address :8888 >/S3/minio.log 2>&1 &" >> /etc/rc.d/rc.local

  [

](https://blog.csdn.net/wangbinaaa/article/details/100132778/)