>10sec aggregation performance in elasticsearch -
i'm new elasticsearch , have setup flow of storing jmx data polled every 10seconds elasticearch index looks this:
{ "jmxmetrics": { "mappings": { "kafkajmx": { "properties": { "@timestamp": { "type": "date" }, "@version": { "type": "keyword" }, "attr_type": { "type": "keyword" }, "host": { "type": "keyword" }, "kafka_broker_id": { "type": "keyword" }, "kafka_clientid": { "type": "keyword" }, "kafka_fetcher_id": { "type": "keyword" }, "kafka_metric_group": { "type": "keyword" }, "kafka_metric_name": { "type": "keyword" }, "kafka_metric_type": { "type": "keyword" }, "kafka_topic": { "type": "keyword" }, "metric_path": { "type": "text" }, "metric_value_number": { "type": "long" }, "metric_value_string": { "type": "keyword" }, "path": { "type": "text" }, "topic_partition": { "type": "keyword" } } } } } } i'm using web portal built in reactjs hit api endpoints in elastic search can populate things like: hosts in drop down, or topics in host selected.
for example, if run query:
post /jmxmetrics/kafkajmx/_search { "size": 0, "aggs" : { "topics": { "filter": { "range": { "@timestamp": { "from": "now-60s" } } }, "aggs": { "topics":{ "terms": { "field": "kafka_topic", "size": 500 } } } } }, "query": { "match": { "host": "host_a" } } } it takes 25seconds return:
{ "took": 24709, "timed_out": false, "_shards": { "total": 3, "successful": 3, "failed": 0 }, "hits": { "total": 1053380207, "max_score": 0, "hits": [] }, "aggregations": { "topics": { "doc_count": 54915, "topics": { "doc_count_error_upper_bound": 189, "sum_other_doc_count": 45098, "buckets": [ { "key": "__consumer_offsets", "doc_count": 503 }, { "key": "topic 1", "doc_count": 196 }, { "key": "topic 2", "doc_count": 193 }, { "key": "topic 3", "doc_count": 142 }, ... ] } } } } any elasticsearch aggregation query seems take 10+ seconds run, , cluster topology looks (running on aws ec2):
3 master/data nodes c4.2xlarge (8 cores, , 15gb ram) 1 client node t2.large (2 cores, 8gb ram)
any idea on improving here? know broad question pointers helpful.
-------edit--------
here settings:
{ "jmxmetrics": { "settings": { "index": { "creation_date": "1489771989403", "number_of_shards": "3", "number_of_replicas": "2", "uuid": "delp9mrqt6s_kilmmxorew", "version": { "created": "5020199" }, "provided_name": "sdpjmx" } } } } also, here stats:
{ "_shards": { "total": 9, "successful": 9, "failed": 0 }, "_all": { "primaries": { "docs": { "count": 1402484967, "deleted": 0 }, "store": { "size_in_bytes": 261239372859, "throttle_time_in_millis": 0 }, "indexing": { "index_total": 1402487151, "index_time_in_millis": 108231827, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 0, "is_throttled": false, "throttle_time_in_millis": 0 }, "get": { "total": 1, "time_in_millis": 13, "exists_total": 0, "exists_time_in_millis": 0, "missing_total": 1, "missing_time_in_millis": 13, "current": 0 }, "search": { "open_contexts": 0, "query_total": 692, "query_time_in_millis": 1390846, "query_current": 0, "fetch_total": 76, "fetch_time_in_millis": 15444, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 4, "current_docs": 57546764, "current_size_in_bytes": 10961768409, "total": 359830, "total_time_in_millis": 298632418, "total_docs": 8639051262, "total_size_in_bytes": 1733171768674, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 208268389, "total_auto_throttle_in_bytes": 15728640 }, "refresh": { "total": 2290338, "total_time_in_millis": 27698495, "listeners": 0 }, "flush": { "total": 1398, "total_time_in_millis": 512144 }, "warmer": { "current": 0, "total": 2291685, "total_time_in_millis": 365893 }, "query_cache": { "memory_size_in_bytes": 131229184, "total_count": 13019, "hit_count": 905, "miss_count": 12114, "cache_size": 118, "cache_count": 139, "evictions": 21 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0 }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 167, "memory_in_bytes": 868415719, "terms_memory_in_bytes": 731359089, "stored_fields_memory_in_bytes": 125593776, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 32064, "points_memory_in_bytes": 11138810, "doc_values_memory_in_bytes": 291980, "index_writer_memory_in_bytes": 9788964, "version_map_memory_in_bytes": 290304, "fixed_bit_set_memory_in_bytes": 0, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 529724, "size_in_bytes": 275528684 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 84 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 0 } }, "total": { "docs": { "count": 4207454779, "deleted": 0 }, "store": { "size_in_bytes": 776849456528, "throttle_time_in_millis": 0 }, "indexing": { "index_total": 4207461365, "index_time_in_millis": 333522565, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 0, "is_throttled": false, "throttle_time_in_millis": 0 }, "get": { "total": 2, "time_in_millis": 13, "exists_total": 0, "exists_time_in_millis": 0, "missing_total": 2, "missing_time_in_millis": 13, "current": 0 }, "search": { "open_contexts": 0, "query_total": 2046, "query_time_in_millis": 4439867, "query_current": 0, "fetch_total": 204, "fetch_time_in_millis": 61910, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 9, "current_docs": 90922106, "current_size_in_bytes": 17436129588, "total": 993305, "total_time_in_millis": 889392309, "total_docs": 25596362951, "total_size_in_bytes": 5109527514241, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 622031369, "total_auto_throttle_in_bytes": 47185920 }, "refresh": { "total": 6870703, "total_time_in_millis": 80442220, "listeners": 0 }, "flush": { "total": 4194, "total_time_in_millis": 1542414 }, "warmer": { "current": 0, "total": 6874724, "total_time_in_millis": 1048456 }, "query_cache": { "memory_size_in_bytes": 428618904, "total_count": 38315, "hit_count": 3196, "miss_count": 35119, "cache_size": 371, "cache_count": 438, "evictions": 67 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0 }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 492, "memory_in_bytes": 2604314152, "terms_memory_in_bytes": 2194198429, "stored_fields_memory_in_bytes": 375756256, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 94464, "points_memory_in_bytes": 33422427, "doc_values_memory_in_bytes": 842576, "index_writer_memory_in_bytes": 22793264, "version_map_memory_in_bytes": 853888, "fixed_bit_set_memory_in_bytes": 0, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 1591047, "size_in_bytes": 827579505 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 345 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 0 } } }, "indices": { "jmxmetrics": { "primaries": { "docs": { "count": 1402484967, "deleted": 0 }, "store": { "size_in_bytes": 261239372859, "throttle_time_in_millis": 0 }, "indexing": { "index_total": 1402487151, "index_time_in_millis": 108231827, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 0, "is_throttled": false, "throttle_time_in_millis": 0 }, "get": { "total": 1, "time_in_millis": 13, "exists_total": 0, "exists_time_in_millis": 0, "missing_total": 1, "missing_time_in_millis": 13, "current": 0 }, "search": { "open_contexts": 0, "query_total": 692, "query_time_in_millis": 1390846, "query_current": 0, "fetch_total": 76, "fetch_time_in_millis": 15444, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 4, "current_docs": 57546764, "current_size_in_bytes": 10961768409, "total": 359830, "total_time_in_millis": 298632418, "total_docs": 8639051262, "total_size_in_bytes": 1733171768674, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 208268389, "total_auto_throttle_in_bytes": 15728640 }, "refresh": { "total": 2290338, "total_time_in_millis": 27698495, "listeners": 0 }, "flush": { "total": 1398, "total_time_in_millis": 512144 }, "warmer": { "current": 0, "total": 2291685, "total_time_in_millis": 365893 }, "query_cache": { "memory_size_in_bytes": 131229184, "total_count": 13019, "hit_count": 905, "miss_count": 12114, "cache_size": 118, "cache_count": 139, "evictions": 21 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0 }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 167, "memory_in_bytes": 868415719, "terms_memory_in_bytes": 731359089, "stored_fields_memory_in_bytes": 125593776, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 32064, "points_memory_in_bytes": 11138810, "doc_values_memory_in_bytes": 291980, "index_writer_memory_in_bytes": 9788964, "version_map_memory_in_bytes": 290304, "fixed_bit_set_memory_in_bytes": 0, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 529724, "size_in_bytes": 275528684 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 84 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 0 } }, "total": { "docs": { "count": 4207454779, "deleted": 0 }, "store": { "size_in_bytes": 776849456528, "throttle_time_in_millis": 0 }, "indexing": { "index_total": 4207461365, "index_time_in_millis": 333522565, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 0, "is_throttled": false, "throttle_time_in_millis": 0 }, "get": { "total": 2, "time_in_millis": 13, "exists_total": 0, "exists_time_in_millis": 0, "missing_total": 2, "missing_time_in_millis": 13, "current": 0 }, "search": { "open_contexts": 0, "query_total": 2046, "query_time_in_millis": 4439867, "query_current": 0, "fetch_total": 204, "fetch_time_in_millis": 61910, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 9, "current_docs": 90922106, "current_size_in_bytes": 17436129588, "total": 993305, "total_time_in_millis": 889392309, "total_docs": 25596362951, "total_size_in_bytes": 5109527514241, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 622031369, "total_auto_throttle_in_bytes": 47185920 }, "refresh": { "total": 6870703, "total_time_in_millis": 80442220, "listeners": 0 }, "flush": { "total": 4194, "total_time_in_millis": 1542414 }, "warmer": { "current": 0, "total": 6874724, "total_time_in_millis": 1048456 }, "query_cache": { "memory_size_in_bytes": 428618904, "total_count": 38315, "hit_count": 3196, "miss_count": 35119, "cache_size": 371, "cache_count": 438, "evictions": 67 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0 }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 492, "memory_in_bytes": 2604314152, "terms_memory_in_bytes": 2194198429, "stored_fields_memory_in_bytes": 375756256, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 94464, "points_memory_in_bytes": 33422427, "doc_values_memory_in_bytes": 842576, "index_writer_memory_in_bytes": 22793264, "version_map_memory_in_bytes": 853888, "fixed_bit_set_memory_in_bytes": 0, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 1591047, "size_in_bytes": 827579505 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 345 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 0 } } } } }
Comments
Post a Comment