ElasticSearch 写入性能压测及优化

硬件信息

第一轮

ES配置

{
  "index": {
    "lifecycle": {
      "name": "realinfo_policy",
      "rollover_alias": "realinfo-index"
    },
    "max_result_window": "500000",
    "refresh_interval": "60s",
    "number_of_shards": "36",
    "number_of_replicas": "1"
  }
}

{
  "_doc": {
    "dynamic_templates": [
      {
        "double_as_float": {
          "mapping": {
            "type": "float"
          },
          "match_mapping_type": "double"
        }
      },
      {
        "strings": {
          "mapping": {
            "type": "keyword"
          },
          "match_mapping_type": "string"
        }
      }
    ],
    "properties": {
      "9999": {
        "format": "yyyy-MM-dd HH:mm:ss ||strict_date_optional_time||epoch_millis",
        "type": "date"
      },
      "VID": {
        "type": "keyword"
      },
      "MESSAGE": {
        "type": "keyword",
        "doc_values": false
      },
      "MESSAGETYPE": {
        "type": "keyword"
      },
      "VIN": {
        "type": "keyword"
      },
      "TIME": {
        "format": "yyyy-MM-dd HH:mm:ss ||strict_date_optional_time||epoch_millis",
        "type": "date"
      }
    }
  }
}

数据样本

数据大小12k

SUBMIT 0 LK6ADAE26LB300894 REALTIME {VID:78e374ca739649839d2951504c67f280,VTYPE:0968059f2f294bc1871b280284c8f7fc,2000:20201105183337,SVRNUM:,CTYPE:1_1_1,2101:16,2102:1,2103:MTo2M182M182M182M182M182M182M182M182M182M182M182M182M182M182M182Mw==,2201:0,2202:18350,2203:15,2204:0,2205:0,2208:0,2209:0,2213:1,2214:1,2301:1,2501:0,2502:109678855,2503:23954715,2601:1,2602:25,2603:3328,2604:1,2605:9,2606:3326,2607:1,2608:1,2609:63,2610:1,2611:1,2612:63,2613:3193,2614:9942,2615:78,2617:5698,2804:0,2805:,2808:0,2809:,2920:0,2921:0,2922:,2923:0,2924:,3201:2,3801:0,7101:16,7102:1,7103:Pz8/Pz8/Pz8/Pz8/Pz8/Pw==,7615:78,2001:96,2002:1,2003:MTozMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyNl8zMzI2XzMzMjZfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyNl8zMzI3XzMzMjdfMzMyNl8zMzI2XzMzMjZfMzMyN18zMzI4XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjZfMzMyN18zMzI3XzMzMjdfMzMyNl8zMzI2XzMzMjZfMzMyNl8zMzI3XzMzMjdfMzMyNl8zMzI2XzMzMjZfMzMyOF8zMzI3XzMzMjZfMzMyNl8zMzI2XzMzMjZfMzMyNl8zMzI3XzMzMjZfMzMyNl8zMzI2XzMzMjZfMzMyN18zMzI4XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyOF8zMzI3XzMzMjdfMzMyNl8zMzI2XzMzMjdfMzMyNl8zMzI3XzMzMjdfMzMyN18zMzI3XzMzMjdfMzMyOF8zMzI3XzMzMjZfMzMyNl8zMzI2XzMzMjZfMzMyNl8zMzI3XzMzMjZfMzMyNl8zMzI2XzMzMjZfMzMyNw==,7001:96,7002:1,7003:DP8M/wz/DP8M/wz/DP8M/wz+DP4M/gz/DP8M/wz/DP8M/wz+DP8M/wz+DP4M/gz/DQAM/wz/DP8M/wz/DP8M/wz/DP8M/gz/DP8M/wz+DP4M/gz+DP8M/wz+DP4M/g0ADP8M/gz+DP4M/gz+DP8M/gz+DP4M/gz/DQAM/wz/DP8M/wz/DP8M/wz/DP8M/w0ADP8M/wz+DP4M/wz+DP8M/wz/DP8M/w0ADP8M/gz+DP4M/gz+DP8M/gz+DP4M/gz/,211111:0,168002:300,211162:0,168003:3514,4672:9942,168004:3209,4674:5698,211304:0,168126:15,5748:248,5749:0,5750:0,168074:245,168006:100,4678:78,168007:7,168008:232,168009:2697,168010:5000,168013:2697,168014:5000,168015:2697,168016:5000,4687:3193,168017:3514,168018:3212,4690:3328,4691:25,4692:3326,4693:9,168019:3326,168023:1,168021:1,168020:1,168027:0,211191:0,168075:1,168024:0,168028:1,168029:0,168030:0,168031:3,168087:73,4708:73,4709:1,4710:73,4711:1,168039:0,168040:0,168088:0,168042:0,168041:0,168044:0,168050:0,168051:0,168052:0,168047:0,168046:0,168045:0,168049:0,168048:0,168043:0,168053:78,168089:320,168054:307,4730:16,4731:26,4732:1,168055:1,168056:1,168057:1,168058:1,168059:1,168060:1,168061:0,168063:3,168062:113,168064:150,4743:73,4744:73,4745:73,4746:73,4747:73,4748:73,4749:73,4750:73,4751:73,4752:73,4753:73,4754:73,4755:73,4756:73,4757:73,4758:73,4759:255,4760:255,4761:255,4762:255,4763:255,4764:255,4765:255,4766:255,211327:255,211328:255,211329:255,211330:255,211331:255,211332:255,211333:255,211334:255,211335:255,211336:255,211337:255,211338:255,211339:255,211340:255,211341:255,211342:255,168076:255,168077:255,168078:255,168079:255,210814:255,210815:255,211345:0,211346:0,211039:1,305941:3327,211112:2,305942:3327,211113:3,305943:3327,211119:4,305944:3327,210777:1,305945:3327,210778:2,305946:3327,210779:3,305947:3327,210780:4,305948:3327,210781:1,305949:3326,210782:2,305950:3326,210783:3,305951:3326,210784:4,305952:3327,210785:1,305953:3327,210786:2,305954:3327,210787:3,305955:3327,210788:4,305956:3327,210789:1,305957:3327,210790:2,305958:3326,210791:3,305959:3327,210792:4,305960:3327,210793:1,305961:3326,210794:2,305962:3326,210795:3,305963:3326,210796:4,305964:3327,210797:1,305965:3328,210798:2,305966:3327,210799:3,305967:3327,210800:4,305968:3327,210801:1,305969:3327,210802:2,305970:3327,210803:3,305971:3327,210804:4,305972:3327,210805:1,305973:3327,210806:2,305974:3327,210807:3,305975:3326,210808:4,305976:3327,210809:1,305977:3327,210810:2,305978:3327,210811:3,211115:3326,210812:4,211116:3326,211048:1,211118:3326,211049:2,211120:3326,211050:3,211114:3327,211051:4,211117:3327,211052:1,211121:3326,211053:2,211122:3326,211054:3,211123:3326,211055:4,211124:3328,211056:1,211125:3327,211057:2,211126:3326,211058:3,211127:3326,211059:4,211128:3326,211060:1,211129:3326,211061:2,211130:3326,211062:3,211131:3327,211063:4,211132:3326,211064:1,211133:3326,211065:2,211134:3326,206023:3,205998:3326,206024:4,211135:3327,211066:1,211136:3328,211067:2,211137:3327,211068:3,211138:3327,211069:4,211139:3327,211070:1,211140:3327,211071:2,211141:3327,211072:3,211142:3327,211073:4,211143:3327,211074:1,211144:3327,211075:2,211145:3327,211076:3,211146:3327,211077:4,211147:3328,211078:1,211148:3327,211079:2,211149:3327,211080:3,211150:3326,211081:4,211151:3326,211082:1,211152:3327,211083:2,211153:3326,211084:3,211154:3327,211085:4,211155:3327,211086:1,211156:3327,211087:2,211157:3327,211088:3,211158:3327,211089:4,211159:3328,211090:1,211160:3327,211091:2,211161:3326,211092:3,211163:3326,211093:4,211164:3326,211094:1,211165:3326,211095:2,211166:3326,211096:3,211167:3327,211097:4,211168:3326,211098:1,211169:3326,211099:2,211170:3326,211100:3,211171:3326,211101:4,211172:3327,211102:7,211173:8191,211103:7,211174:8191,210823:7,211175:8191,210824:7,211176:8191,211040:7,211177:8191,211041:7,211178:8191,211042:7,211179:8191,211043:7,211180:8191,211044:7,211181:8191,211045:7,211182:8191,211046:7,211183:8191,211047:7,211184:8191,211104:7,211185:8191,211105:7,211186:8191,211106:7,211187:8191,211107:7,211188:8191,168072:3158856,168065:80,168066:66,168067:3158613,168068:52983525098309,168069:4273492,168070:13563782407139894,211189:0,211190:0,305919:4000,305920:4000,211192:0,211193:0,211194:0,211195:0,211196:0,211197:0,211198:0,211199:0,211200:0,211201:0,211202:0,211203:0,211204:0,211205:0,211206:0,211207:0,211208:3193,211343:3942,211209:0,211344:4000,211210:0,211211:0,211212:1,211213:0,211214:0,211215:0,211216:0,211217:0,211218:0,211219:0,211220:0,211221:0,211222:0,211223:0,211224:0,211225:0,211226:0,211227:0,211228:0,211229:0,211230:0,211231:0,211232:0,211233:0,211234:0,211235:0,211236:0,211237:0,211238:0,4815:1,143001:0,143052:0,143044:2,143002:2044,211239:0,211240:0,211241:0,143045:16384,140070:15,5899:84,168121:2,140001:4,4577:2,168106:1,124026:0,124028:0,143004:2044,121011:3,143003:255,4585:22,168107:0,140013:2,140014:1,168105:1,140067:0,180000:1,140063:0,140015:0,140007:0,140006:0,140054:0,140059:0,140008:237,140002:9942,140004:3212,168092:0,143046:0,140016:0,168098:0,140011:0,140010:0,143047:0,140017:0,140018:0,143048:0,181020:0,181021:0,168108:0,121014:0,181022:0,168125:0,140012:78,140055:0,4616:0,5947:0,143043:16384,5949:2,165026:0,167002:0,140009:0,140062:0,5954:14,211242:0,165072:1,181001:0,183020:0,140020:0,180001:1391,140021:2,165051:0,183021:1,140022:0,140023:0,183002:0,183003:0,183004:0,183005:0,305997:1,166001:1,190002:1020,305924:0,167057:0,143049:0,146003:0,146004:0,140038:0,140024:0,140025:0,140026:0,140027:0,140028:0,140029:0,140030:0,140031:0,140032:0,4649:1,211247:0,143011:0,143010:1,143042:2,4653:2044,4654:16384,143013:1020,146006:3,5998:205,211326:2044,211248:0,143016:1023,143017:320,143005:0,143006:0,143007:0,6006:3,6007:178,143008:0,143009:0,211249:0,211250:0,211251:0,4661:78,4662:83,143020:83,6013:6,6014:250,143025:0,206022:0,300395:0,211252:8330245,211253:3,211254:30,211255:24,300396:0,300397:0,4669:1,211260:2,165003:51,165004:3188,165005:252,206021:87,165008:8,165009:9,211261:0,165010:0,165011:0,165012:1,165016:2,165017:2,165018:0,165019:64,165020:64,165007:0,211262:86,180004:0,180005:0,180006:0,180007:0,180008:0,180009:0,211263:0,211264:0,211265:0,211266:0,211267:0,211268:84,206025:5,165013:0,210813:19,211269:140,211270:146,211271:3188,211272:51,211273:0,211274:0,211275:0,211276:0,211277:1,211278:7,211279:153,165030:0,211280:0,211281:0,211282:0,165023:3188,211283:0,165022:51,211284:0,211285:0,165024:2524,165025:87,4833:0,140033:117440,140034:0,211289:0,211290:0,4836:1,211291:0,211292:0,211293:1,211245:7,211246:0,305926:165,5012:1,122009:0,5014:0,122010:0,122011:0,122012:0,122013:0,122014:0,122015:0,122016:0,122017:0,122018:0,5024:1,121002:0,180010:0,107016:0,185005:0,141030:0,141031:0,181008:0,107020:0,107021:0,107022:0,123012:0,140060:0,107017:0,123013:0,211295:0,300398:0,181009:0,211038:0,140040:0,140041:0,140042:0,121009:0,211298:0,211299:0,140045:0,107008:0,140047:0,140048:0,140049:0,140050:0,140051:0,107018:0,211300:0,211301:0,107009:0,107010:0,140052:0,140039:0,140056:0,124030:0,121003:0,181018:0,107011:0,124031:0,105001:0,105002:0,181017:0,181016:0,181015:0,300399:0,181014:0,211302:0,181013:0,181011:0,181012:0,6178:1,300400:0,5648:0,107012:0,104001:0,305979:0,107007:0,210895:0,4888:0,107001:0,5655:0,107002:0,107004:1,107005:1,107003:1,107006:0,108003:1,6187:0,6188:0,210894:0,113001:0,113002:0,113003:0,113004:0,113005:0,6194:0,6195:0,6196:0,6197:0,6198:0,6199:0,210893:0,210892:0,4985:0,183007:1,6202:0,183008:0,183009:0,183010:0,183011:0,183012:1,210891:255,183013:0,183014:0,183015:0,183016:0,183017:0,183018:0,183019:0,211303:0,6215:1,210883:0,210884:0,210885:0,121004:0,210886:0,124037:0,120003:0,120001:0,186002:0,120002:0,210887:0,210888:0,210889:1,210890:1,124038:0,124011:0,120004:0,210876:0,210875:0,124039:3,120011:0,210877:12,210878:0,210879:0,120005:0,210881:0,210882:0,120007:0,120008:0,120009:0,120010:0,6274:0,6275:0,122002:0,122004:0,4854:3,4855:3,122006:0,122008:0,140053:3,4861:3,305980:0,305981:0,305982:1,305983:1,305984:0,305985:0,305986:1,305987:1,210871:0,120006:0,124015:0,210873:0,210874:0,210858:3,210866:3,210867:15,305988:0,210868:3,210869:555,210870:44,211308:3,210859:3,210860:0,210861:817,210862:3,210863:0,210864:330,210865:111,210857:0,124002:0,124004:0,124005:0,124007:0,6219:0,305989:0,124014:0,124013:0,305990:0,124008:0,6227:0,6298:1,211013:0,211015:0,300402:0,211017:1,211033:1,300406:0,211035:41,6228:1,210851:0,124016:0,124017:0,124018:0,124019:0,124020:0,143040:0,143041:0,124021:0,210852:0,124023:0,210853:0,124024:0,124025:0,210854:0,210855:0,210856:0,210847:0,210848:0,210849:0,210850:0,210846:1,123001:2,123002:1,123003:65509,123006:0,123004:0,123007:1,4869:10,4870:201,210837:0,210838:0,210839:0,210840:0,210841:0,210842:0,206005:0,210843:0,210844:0,210845:0,4871:1,6238:0,6239:0,210834:0,210835:0,210836:0,185003:0,123008:0,123009:0,6243:1,111002:0,6245:0,185001:0,185002:0,108005:0,108006:1,108008:0,108007:0,111005:6,6255:11,4844:1,122022:0,210998:0,210999:0,211000:0,211001:0,113015:0,184001:0,211002:0,205971:0,211003:0,113016:0,113017:0,113018:0,210994:0,113020:0,113019:0,210995:0,113021:0,113022:0,210996:0,210997:0,113023:0,210988:0,210989:0,210990:0,210991:0,210992:0,210993:0,205987:0,205985:0,205986:0,210984:1,210972:0,210973:0,210974:0,210975:0,210976:0,210977:0,210978:0,210979:0,210980:0,210981:0,210982:0,210983:0,210962:0,210963:0,210964:0,210965:0,210966:0,210967:0,210968:0,210969:0,210970:0,210971:0,210945:0,210946:0,210947:0,210948:0,210949:0,210950:0,210951:0,210952:0,210953:0,210954:0,210955:0,210956:0,210957:0,210958:0,210959:0,210960:0,210961:0,210943:0,210944:0,210925:0,210926:0,210927:0,210928:0,210929:0,210930:0,210934:0,210935:0,210936:0,210937:0,210938:0,210939:0,210940:0,210941:0,305991:0,305992:0,210924:1,210914:0,210915:0,210916:0,210917:0,210918:0,210919:0,210920:0,210921:0,210922:0,210923:0,210906:0,210907:0,210908:0,210909:0,210910:0,210911:0,210912:0,210913:0,210898:0,210899:0,210900:0,210901:0,210902:0,210903:0,210904:0,210905:0,210897:1,205999:0,206000:0,206001:297,206002:1003,211256:0,211257:0,211258:0,211259:0,167001:1,4559:0,211286:0,168122:0,168123:0,113006:0,113007:0,113008:0,113009:0,4565:0,211287:0,300423:0,300424:0,300425:0,300426:0,300427:0,113010:0,113011:0,113012:0,113013:0,5732:0,5733:0,5734:0,5735:0,300413:0,300414:0,300415:0,300416:0,300417:0,300418:0,300419:0,300420:0,300421:0,300393:0,305931:0,210828:0,305938:0,305939:0,4414:1119,4415:252,4416:7493,5031:12,300429:197,4506:0,5037:0,5030:20,5029:11,5028:5,5027:18,5026:33,5025:38,9999:20201105183339}

压测结果

总结

1. 3台机器只能处理每秒70多M的数据写入，1个副本的话只能维持不足3000的tps
2. 磁盘IO等待很高，IOPS很高，需要想办法降低
3. 内存使用和CPU使用都在正常范围内
4. 所以要想提高性能需要尝试优化IOPS，段合并、单条数据体积，压缩等

第二轮

优化translog，降低IOPS

translog写入了一份全量的数据，它有点像MysSQL中的binlog，或者redis的aof，用来保证异常情况下的数据安全。
这是因为，我们把数据写到磁盘后，还要调用fsync才能把数据刷到磁盘中，如果不这样做在系统掉电的时候就会导致数据丢失。
ES默认每次请求都进行一次flush，但对于日志来说，这没有必要，可以将这个过程改为异步的，参数如下：

{
  "index.translog.durability" : "async",
  "index.translog.flush_threshold_size" : "512mb",
  "index.translog.sync_interval" : "60s"
}

这可以说是最重要的一步优化了，对性能的影响最大，但在极端情况下会有丢失部分数据的可能。对于日志系统来说，是可以忍受的。

优化段合并并行度

index.merge.scheduler.max_thread_count默认设置为
Math.max(1, Math.min(4, Runtime.getRuntime().availableProcessors() / 2))
但这适用于SSD配置。对于HDD，应将其设置为1。

curl -XPUT 'localhost:9200/_settings' -d '{ 
     "index.merge.scheduler.max_thread_count" : 1
}

压测结果

吞吐量保持不变的情况下


开始增加并行度，写入更大量的数据把富余资源利用上，吞吐量提升到平均4500TPS

负载大幅度降低

总结

• translog异步化可以大幅降低IOPS，从而提升吞吐量

  第三轮

  优化写入buffer

  修改es配置文件
  vim /etc/elasticsearch/elasticsearch.yml

  indices.memory.index_buffer_size: 15%
  thread_pool.write.queue_size: 1024
  

  压测结果

  性能没有明显变化，磁盘性能瓶颈

  第四轮

  优化schema

  “index”: false 不需要检索的字段设置不做倒排索引
  “doc_values”: false 不需要聚合的字段不做列存

  {
  "_doc": {
    "dynamic_templates": [
      {
        "double_as_float": {
          "mapping": {
            "type": "float"
          },
          "match_mapping_type": "double"
        }
      },
      {
        "strings": {
          "mapping": {
            "type": "keyword",
            "doc_values": false,
            "index": false
          },
          "match_mapping_type": "string"
        }
      }
    ],
    "properties": {
      "9999": {
        "format": "yyyy-MM-dd HH:mm:ss ||strict_date_optional_time||epoch_millis",
        "type": "date"
      },
      "VID": {
        "type": "keyword"
      },
      "MESSAGE": {
        "type": "keyword",
        "doc_values": false,
        "index": false
      },
      "MESSAGETYPE": {
        "type": "keyword"
      },
      "VIN": {
        "type": "keyword"
      },
      "TIME": {
        "format": "yyyy-MM-dd HH:mm:ss ||strict_date_optional_time||epoch_millis",
        "type": "date"
      }
    }
  }
  }
  

  压测结果

  大概 TPS 达到了5100
  
  延迟更低
  
  负载更低
  

第五轮

开启最佳压缩

“index.codec”: “best_compression”

测试结果

TPS 提升到5500

第六轮

增加写入并行度

concurrent.request: 12
负载和CPU都有所降低，可以尝试增加并行度，看看资源利用率是不是可以进一步提高

测试结果

TPS 达到5800，负载正常

第七轮

更改schema，单字段变为多字段

测试结果

性能有所降低 只有5200 TPS，但是存储占用只有单字段的一半左右，待后续详细测试对比，看ES是否推荐这么多的字段存储

第八轮

减小数据样本

生产环境并不是每条数据都是12k，按平均8k 来模拟数据样本进行测试

测试结果

TPS 达到8300，和数据降本减小的比率吻合

结论