{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "50GB",
            "max_age": "7d",
            "max_docs": 10000000
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "allocate": {
            "number_of_replicas": 0
          },
          "forcemerge": {
            "max_num_segments": 1
          },
          "set_priority": {
            "priority": 50
          }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": {
            "include": {
              "box_type": "cold"
            }
          },
          "set_priority": {
            "priority": 0
          }
        }
      },
      "delete": {
        "min_age": "365d"
      }
    }
  }
}

{
  "mappings": {
    "dynamic": "true",
    "dynamic_templates": [
      {
        "strings_as_keywords": {
          "match_mapping_type": "string",
          "match": "*_id",
          "mapping": {
            "type": "keyword"
          }
        }
      },
      {
        "strings_as_text": {
          "match_mapping_type": "string",
          "match": "*_name",
          "mapping": {
            "type": "text",
            "analyzer": "ik_max_word"
          }
        }
      }
    ]
  }
}

{
  "settings": {
    "analysis": {
      "analyzer": {
        "ik_smart_analyzer": {
          "type": "ik_smart"
        },
        "ik_max_word_analyzer": {
          "type": "ik_max_word"
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "title": {
        "type": "text",
        "analyzer": "ik_max_word",
        "search_analyzer": "ik_smart"
      },
      "content": {
        "type": "text",
        "analyzer": "ik_max_word"
      }
    }
  }
}

{
  "query": {
    "bool": {
      "must": [
        {"match": {"title": "elasticsearch"}}
      ],
      "filter": [
        {"term": {"status": "published"}},
        {"range": {"publish_date": {"gte": "2023-01-01"}}}
      ],
      "must_not": [
        {"term": {"category": "draft"}}
      ],
      "should": [
        {"match": {"tags": "search"}},
        {"match": {"tags": "analytics"}}
      ],
      "minimum_should_match": 1
    }
  }
}

{
  "query": {
    "multi_match": {
      "query": "elasticsearch 搜索引擎",
      "fields": ["title^3", "content^1", "tags^2"],
      "type": "best_fields",
      "tie_breaker": 0.3,
      "minimum_should_match": "75%",
      "analyzer": "ik_smart"
    }
  }
}

{
  "query": {
    "function_score": {
      "query": {"match": {"title": "elasticsearch"}},
      "functions": [
        {
          "filter": {"term": {"category": "tutorial"}},
          "weight": 2
        },
        {
          "field_value_factor": {
            "field": "popularity",
            "factor": 1.2,
            "modifier": "sqrt",
            "missing": 1
          }
        },
        {
          "gauss": {
            "publish_date": {
              "origin": "now",
              "scale": "30d",
              "decay": 0.5
            }
          }
        }
      ],
      "score_mode": "multiply",
      "boost_mode": "multiply",
      "max_boost": 5.0
    }
  }
}

{
  "query": {
    "nested": {
      "path": "comments",
      "query": {
        "bool": {
          "must": [
            {"match": {"comments.author": "张三"}},
            {"range": {"comments.date": {"gte": "2023-01-01"}}}
          ]
        }
      },
      "inner_hits": {
        "highlight": {
          "fields": {
            "comments.content": {}
          }
        },
        "size": 3
      }
    }
  }
}

{
  "size": 0,
  "aggs": {
    "sales_by_category": {
      "terms": {
        "field": "category.keyword",
        "size": 10,
        "order": {"total_revenue": "desc"}
      },
      "aggs": {
        "monthly_trend": {
          "date_histogram": {
            "field": "sale_date",
            "calendar_interval": "month",
            "format": "yyyy-MM"
          },
          "aggs": {
            "total_revenue": {
              "sum": {"field": "revenue"}
            },
            "avg_price": {
              "avg": {"field": "price"}
            },
            "order_count": {
              "value_count": {"field": "order_id"}
            },
            "revenue_growth": {
              "derivative": {
                "buckets_path": "total_revenue"
              }
            }
          }
        },
        "total_revenue": {
          "sum": {"field": "revenue"}
        },
        "price_distribution": {
          "histogram": {
            "field": "price",
            "interval": 100
          }
        }
      }
    }
  }
}

{
  "aggs": {
    "user_behavior_stats": {
      "stats": {
        "field": "session_duration"
      }
    },
    "session_percentiles": {
      "percentiles": {
        "field": "session_duration",
        "percents": [25, 50, 75, 90, 95, 99]
      }
    },
    "unique_users": {
      "cardinality": {
        "field": "user_id",
        "precision_threshold": 10000
      }
    },
    "top_pages": {
      "terms": {
        "field": "page_url.keyword",
        "size": 10
      },
      "aggs": {
        "sample_visits": {
          "top_hits": {
            "size": 3,
            "_source": ["user_id", "timestamp", "referrer"],
            "sort": [{"timestamp": {"order": "desc"}}]
          }
        }
      }
    }
  }
}

{
  "aggs": {
    "sample": {
      "sampler": {
        "shard_size": 1000
      },
      "aggs": {
        "popular_tags": {
          "terms": {
            "field": "tags.keyword",
            "size": 10
          }
        }
      }
    }
  }
}

{
  "settings": {
    "index": {
      "refresh_interval": "30s",
      "number_of_replicas": 0,
      "translog": {
        "durability": "async",
        "sync_interval": "5s",
        "flush_threshold_size": "1gb"
      },
      "merge": {
        "policy": {
          "max_merge_at_once": 30,
          "segments_per_tier": 30
        }
      },
      "codec": "best_compression"
    }
  }
}

# 堆内存设置（不超过32GB，建议物理内存的50%）
-Xms16g
-Xmx16g

# GC算法选择（推荐G1GC）
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
-XX:G1HeapRegionSize=16m

# GC日志配置
-XX:+PrintGC
-XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-XX:+PrintGCApplicationStoppedTime
-Xloggc:/var/log/elasticsearch/gc.log
-XX:+UseGCLogFileRotation
-XX:NumberOfGCLogFiles=10
-XX:GCLogFileSize=100M

# 内存优化
-XX:+UseLargePages
-XX:LargePageSizeInBytes=2m
-XX:+AlwaysPreTouch

# 其他优化参数
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true

#!/bin/bash
# Elasticsearch集群健康检查脚本

ES_HOST="${ES_HOST:-localhost:9200}"
ALERT_EMAIL="${ALERT_EMAIL:-[email protected]}"
LOG_FILE="/var/log/es-health-check.log"

# 日志函数
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# 检查集群状态
check_cluster_health() {
    local health_response=$(curl -s "${ES_HOST}/_cluster/health")
    local status=$(echo "$health_response" | jq -r '.status')
    local active_shards=$(echo "$health_response" | jq -r '.active_shards')
    local unassigned_shards=$(echo "$health_response" | jq -r '.unassigned_shards')
    
    case $status in
        "green")
            log "✅ 集群状态正常 - 活跃分片: $active_shards"
            return 0
            ;;
        "yellow")
            log "⚠️  集群状态警告 - 未分配分片: $unassigned_shards"
            return 1
            ;;
        "red")
            log "❌ 集群状态异常 - 状态: $status"
            return 2
            ;;
        *)
            log "❓ 无法获取集群状态"
            return 3
            ;;
    esac
}

# 检查节点状态
check_nodes_status() {
    local nodes_info=$(curl -s "${ES_HOST}/_cat/nodes?format=json")
    local total_nodes=$(echo "$nodes_info" | jq length)
    local master_nodes=$(echo "$nodes_info" | jq '[.[] | select(.master == "*")] | length')
    
    log "📊 节点状态 - 总节点: $total_nodes, 主节点: $master_nodes"
    
    if [ "$master_nodes" -eq 0 ]; then
        log "❌ 没有活跃的主节点"
        return 1
    fi
    
    return 0
}

# 检查磁盘使用率
check_disk_usage() {
    local disk_info=$(curl -s "${ES_HOST}/_cat/allocation?format=json")
    local high_usage_nodes=$(echo "$disk_info" | jq '[.[] | select(.disk.percent | tonumber > 85)] | length')
    
    if [ "$high_usage_nodes" -gt 0 ]; then
        log "⚠️  发现 $high_usage_nodes 个节点磁盘使用率超过85%"
        return 1
    fi
    
    log "✅ 磁盘使用率正常"
    return 0
}

# 发送告警邮件
send_alert() {
    local subject="Elasticsearch集群告警"
    local body="检测到Elasticsearch集群异常，请及时处理。详细信息请查看日志: $LOG_FILE"
    
    echo "$body" | mail -s "$subject" "$ALERT_EMAIL"
    log "📧 告警邮件已发送到: $ALERT_EMAIL"
}

# 主检查流程
main() {
    log "🚀 开始Elasticsearch集群健康检查"
    
    local exit_code=0
    
    check_cluster_health || exit_code=$?
    check_nodes_status || exit_code=$?
    check_disk_usage || exit_code=$?
    
    if [ $exit_code -ne 0 ]; then
        log "⚠️  发现问题，发送告警通知"
        send_alert
    else
        log "✅ 集群健康检查完成，一切正常"
    fi
    
    exit $exit_code
}

# 执行主函数
main "$@"

# elasticsearch.yml 安全配置
xpack.security.enabled: true

# 传输层安全
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

# HTTP层安全
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: elastic-certificates.p12
xpack.security.http.ssl.truststore.path: elastic-certificates.p12

# 审计日志配置
xpack.security.audit.enabled: true
xpack.security.audit.logfile.events.include:
  - access_denied
  - access_granted
  - anonymous_access_denied
  - authentication_failed
  - connection_denied
  - tampered_request
  - run_as_denied
  - run_as_granted

# IP过滤
xpack.security.http.filter.allow: "192.168.1.0/24,10.0.0.0/8"
xpack.security.http.filter.deny: "_all"

{
  "read_only_analyst": {
    "cluster": ["monitor"],
    "indices": [
      {
        "names": ["logs-*", "metrics-*"],
        "privileges": ["read", "view_index_metadata"],
        "field_security": {
          "grant": ["@timestamp", "message", "level", "service"],
          "except": ["sensitive_data", "personal_info"]
        },
        "query": {
          "match": {
            "department": "analytics"
          }
        }
      }
    ]
  },
  "data_engineer": {
    "cluster": ["monitor", "manage_index_templates"],
    "indices": [
      {
        "names": ["data-*", "etl-*"],
        "privileges": ["read", "write", "create_index", "delete_index", "manage"],
        "field_security": {
          "grant": ["*"]
        }
      }
    ]
  },
  "admin_user": {
    "cluster": ["all"],
    "indices": [
      {
        "names": ["*"],
        "privileges": ["all"]
      }
    ]
  }
}

# 创建API密钥
POST /_security/api_key
{
  "name": "data_ingestion_key",
  "role_descriptors": {
    "data_writer": {
      "cluster": ["monitor"],
      "index": [
        {
          "names": ["logs-*"],
          "privileges": ["write", "create_index"]
        }
      ]
    }
  },
  "expiration": "30d"
}

# 查看API密钥信息
GET /_security/api_key

# 撤销API密钥
DELETE /_security/api_key
{
  "ids": ["api_key_id"]
}

# 1. 查看集群健康状态
GET /_cluster/health?pretty

# 2. 查看未分配分片详情
GET /_cat/shards?v&h=index,shard,prirep,state,unassigned.reason

# 3. 分析分片分配失败原因
GET /_cluster/allocation/explain
{
  "index": "problem_index",
  "shard": 0,
  "primary": false
}

# 4. 查看集群设置
GET /_cluster/settings?include_defaults=true&flat_settings=true

# 1. 启用慢查询日志
PUT /_all/_settings
{
  "index.search.slowlog.threshold.query.warn": "10s",
  "index.search.slowlog.threshold.query.info": "5s",
  "index.search.slowlog.threshold.query.debug": "2s",
  "index.search.slowlog.threshold.query.trace": "500ms",
  "index.search.slowlog.threshold.fetch.warn": "1s",
  "index.search.slowlog.threshold.fetch.info": "800ms",
  "index.search.slowlog.threshold.fetch.debug": "500ms",
  "index.search.slowlog.threshold.fetch.trace": "200ms"
}

# 2. 查看热点线程
GET /_nodes/hot_threads

# 3. 分析查询性能
GET /_nodes/stats/indices/search

# 4. 使用Profile API分析查询
GET /my_index/_search
{
  "profile": true,
  "query": {
    "match": {"title": "elasticsearch"}
  }
}

# 1. 查看JVM内存使用情况
GET /_nodes/stats/jvm?pretty

# 2. 查看GC统计信息
GET /_nodes/stats/jvm?filter_path=nodes.*.jvm.gc

# 3. 检查断路器状态
GET /_nodes/stats/breaker?pretty

# 4. 查看字段数据缓存使用情况
GET /_nodes/stats/indices/fielddata?fields=*

# 1. 创建快照仓库
PUT /_snapshot/backup_repository
{
  "type": "fs",
  "settings": {
    "location": "/backup/elasticsearch",
    "compress": true,
    "chunk_size": "1gb"
  }
}

# 2. 创建快照策略
PUT /_slm/policy/daily_backup
{
  "schedule": "0 2 * * *",
  "name": "<daily-backup-{now/d}>",
  "repository": "backup_repository",
  "config": {
    "indices": ["logs-*", "metrics-*"],
    "ignore_unavailable": true,
    "include_global_state": false
  },
  "retention": {
    "expire_after": "30d",
    "min_count": 7,
    "max_count": 50
  }
}

# 3. 恢复快照
POST /_snapshot/backup_repository/daily-backup-2024.01.01/_restore
{
  "indices": "logs-2024.01.01",
  "ignore_unavailable": true,
  "include_global_state": false,
  "rename_pattern": "logs-(.+)",
  "rename_replacement": "restored-logs-$1"
}

{
  "settings": {
    "number_of_shards": 6,
    "number_of_replicas": 1,
    "refresh_interval": "30s",
    "merge.policy.max_merge_at_once": 30,
    "codec": "best_compression"
  }
}

{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "50GB",
            "max_age": "7d",
            "max_docs": 10000000
          },
          "set_priority": {"priority": 100}
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "allocate": {"number_of_replicas": 0},
          "forcemerge": {"max_num_segments": 1},
          "set_priority": {"priority": 50}
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": {
            "include": {"box_type": "cold"}
          },
          "set_priority": {"priority": 0}
        }
      },
      "delete": {
        "min_age": "365d"
      }
    }
  }
}

{
  "settings": {
    "refresh_interval": "30s",           // 降低刷新频率
    "number_of_replicas": 0,             // 写入时禁用副本
    "translog.durability": "async",      // 异步事务日志
    "translog.sync_interval": "5s",      // 事务日志同步间隔
    "merge.policy.max_merge_at_once": 30,
    "merge.policy.segments_per_tier": 30
  }
}

{
  "settings": {
    "refresh_interval": "1s",
    "number_of_replicas": 1,
    "translog.durability": "request"
  }
}

{
  "query": {
    "bool": {
      "filter": [  // 使用filter context，可缓存
        {"term": {"status": "published"}},
        {"range": {"date": {"gte": "2023-01-01"}}}
      ],
      "must": [    // 只在需要评分时使用
        {"match": {"title": "elasticsearch"}}
      ]
    }
  },
  "size": 20,      // 限制返回结果数量
  "_source": ["title", "summary"],  // 只返回需要的字段
  "sort": [{"date": {"order": "desc"}}]
}

# 查看集群健康状态
GET /_cluster/health?pretty

# 查看具体哪些索引受影响
GET /_cluster/health?level=indices

# 查看分片状态
GET /_cat/shards?v&h=index,shard,prirep,state,node,unassigned.reason

# 手动分配副本分片为主分片
POST /_cluster/reroute
{
  "commands": [
    {
      "allocate_empty_primary": {
        "index": "problem_index",
        "shard": 0,
        "node": "available_node",
        "accept_data_loss": true
      }
    }
  ]
}

# 从快照恢复（如果有备份）
POST /_snapshot/backup_repo/snapshot_name/_restore
{
  "indices": "problem_index",
  "ignore_unavailable": true
}

# 查看JVM内存使用情况
GET /_nodes/stats/jvm?pretty

# 查看GC统计信息
GET /_nodes/stats/jvm?filter_path=nodes.*.jvm.gc

# 检查断路器状态
GET /_nodes/stats/breaker?pretty

# 查看字段数据缓存使用
GET /_nodes/stats/indices/fielddata?fields=*

# 启用详细GC日志
-XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-XX:+PrintGCApplicationStoppedTime
-Xloggc:/var/log/elasticsearch/gc.log

# 分析GC模式
# 正常：Minor GC频繁，Major GC偶尔
# 异常：Major GC频繁，Full GC持续时间长

# 设置合适的堆内存（不超过32GB）
-Xms16g -Xmx16g

# 使用G1GC，适合大堆内存
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200

# 启用压缩指针，节省内存
-XX:+UseCompressedOops

{
  "persistent": {
    "indices.fielddata.cache.size": "40%",
    "indices.queries.cache.size": "10%",
    "indices.requests.cache.size": "1%",
    "indices.breaker.fielddata.limit": "40%",
    "indices.breaker.request.limit": "60%",
    "indices.breaker.total.limit": "95%"
  }
}

# 查看分片分布情况
GET /_cat/shards?v&s=node

# 查看节点分配情况
GET /_cat/allocation?v

# 查看集群分配设置
GET /_cluster/settings?include_defaults=true

{
  "persistent": {
    "cluster.routing.rebalance.enable": "all",
    "cluster.routing.allocation.allow_rebalance": "indices_all_active",
    "cluster.routing.allocation.cluster_concurrent_rebalance": 2,
    "cluster.routing.allocation.balance.shard": 0.45,
    "cluster.routing.allocation.balance.index": 0.55,
    "cluster.routing.allocation.balance.threshold": 1.0
  }
}

# 手动移动分片
POST /_cluster/reroute
{
  "commands": [
    {
      "move": {
        "index": "my_index",
        "shard": 0,
        "from_node": "overloaded_node",
        "to_node": "underutilized_node"
      }
    }
  ]
}

{
  "persistent": {
    "cluster.routing.allocation.exclude._ip": "192.168.1.100",
    "cluster.routing.allocation.include.box_type": "hot,warm",
    "cluster.routing.allocation.require.zone": "zone1"
  }
}

{
  "settings": {
    "similarity": {
      "custom_bm25": {
        "type": "BM25",
        "k1": 1.5,    // 增加词频影响
        "b": 0.5      // 减少长度影响
      }
    }
  },
  "mappings": {
    "properties": {
      "title": {
        "type": "text",
        "similarity": "custom_bm25"
      }
    }
  }
}