部署elasticsearch错误
原创大约 4 分钟
问题
场景
eck operator部署 elasticsearch
配置
elasticsearch-statefulset.yaml
# 测试的话记得把下面相关参数值调低
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: quickstart
namespace: es
spec:
version: 8.11.3
nodeSets:
- name: default # sts名称就是quickstart-es-default
count: 1 # es集群数量,因为测试环境资源较少,生产可以部署2-3个
config:
node.store.allow_mmap: false
podTemplate:
spec:
containers:
- name: elasticsearc
image: easzlab.io.local:5000/elasticsearch/elasticsearch:8.11.3
imagePullPolicy: IfNotPresent # 或者 Never
env:
- name: ES_JAVA_OPTS # es用java开发的
value: -Xms1g -Xmx1g # 资源是分配的1/2,生产需要2G-4G
resources:
requests:
memory: 1.8Gi
cpu: 0.5
limits:
memory: 1.8Gi
cpu: 0.5
initContainers:
- name: sysctl
securityContext:
privileged: true
# 初始化容器,配置内核参数
command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi # 50G可以自己适当分配
storageClassName: nfs-boge
http:
tls:
selfSignedCertificate:
disabled: true运行
# kubectl create ns es
# kubectl -n es apply -f elasticsearch-statefulset.yaml
elasticsearch.elasticsearch.k8s.elastic.co/quickstart created问题现象
pod处于CrashLoopBackOff状态
# kubectl -n es get pod
NAME READY STATUS RESTARTS AGE
quickstart-es-default-0 1/2 CrashLoopBackOff 16 (4m11s ago) 9h排查
describe
# kubectl -n es describe pod quickstart-es-default-0
Init Containers:
elastic-internal-init-filesystem:
Ready: True
elastic-internal-suspend:
Ready: True
sysctl:
Ready: True
Containers:
elasticsearc:
Ready: False
Mounts:
/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
elasticsearch:
Ready: False
Mounts:
/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
Volumes:
elasticsearch-data:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: elasticsearch-data-quickstart-es-default-0
ReadOnly: false
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Created 10m (x2 over 10m) kubelet Created container elasticsearc
Normal Started 10m (x2 over 10m) kubelet Started container elasticsearc
Warning Unhealthy 10m kubelet Readiness probe failed: command "bash -c /mnt/elastic-internal/scripts/readiness-probe-script.sh" timed out
Warning Unhealthy 7m47s (x32 over 9m45s) kubelet (combined from similar events): Readiness probe failed: {"timestamp": "2024-06-15T01:33:47+00:00", "message": "readiness probe failed", "curl_rc": "7"}
Warning BackOff 2m40s (x48 over 10m) kubelet Back-off restarting failed container elasticsearc in pod quickstart-es-default-0_es(1e224a31-7b91-4813-825c-b93fe6e677fb)logs
# kubectl -n es logs quickstart-es-default-0
Defaulted container "elasticsearc" out of: elasticsearc, elasticsearch, elastic-internal-init-filesystem (init), elastic-internal-suspend (init), sysctl (init)
/usr/local/bin/docker-entrypoint.sh: line 84: cannot create temp file for here-document: Read-only file system/usr/local/bin/docker-entrypoint.sh:第84行：无法为此处文档创建临时文件：只读文件系统
/usr/local/bin/docker-entrypoint.sh的第84行代码：
82 # Signal forwarding and child reaping is handled by `tini`, which is the
83 # actual entrypoint of the container
84 exec /usr/share/elasticsearch/bin/elasticsearch "$@" $POSITIONAL_PARAMETERS <<<"$KEYSTORE_PASSWORD"在describe里看/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
# kubectl -n es logs quickstart-es-default-0 -c sysctl
sysctl: permission denied on key "vm.max_map_count", ignoringsc pv pvc
# kubectl -n es get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-boge nfs-provisioner-01 Retain Immediate false 19h
# kubectl -n es get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-506a6c18-4c6a-4bae-a2de-0d42b7c953a4 5Gi RWO Retain Bound es/elasticsearch-data-quickstart-es-default-0 nfs-boge 10h
# kubectl -n es get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
elasticsearch-data-quickstart-es-default-0 Bound pvc-506a6c18-4c6a-4bae-a2de-0d42b7c953a4 5Gi RWO nfs-boge 10h测试
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: new-nginx
namespace: es
labels:
app: new-nginx
spec:
serviceName: "nginx"
replicas: 2
selector:
matchLabels:
app: new-nginx
template:
metadata:
labels:
app: new-nginx
spec:
containers:
- name: new-nginx
image: nginx:1.21.6
env:
- name: TZ
value: Asia/Shanghai
ports:
- containerPort: 80
volumeMounts:
- name: test-data
mountPath: "/usr/share/nginx/html"
- name: busybox
image: registry.cn-shanghai.aliyuncs.com/acs/busybox:v1.29.2
args:
- /bin/sh
- -c
- >
while :; do
if [ -f /html/index.html ];then
echo "[$(date +%F\ %T)] ${MY_POD_NAMESPACE}-${MY_POD_NAME}-${MY_POD_IP}" > /html/index.html
sleep 1
else
touch /html/index.html
fi
done
env:
- name: TZ
value: Asia/Shanghai
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MY_POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: MY_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
volumeMounts:
- name: test-data
mountPath: "/html"
- mountPath: /etc/localtime
name: tz-config
volumes:
- name: tz-config
hostPath:
path: /etc/localtime
type: File
volumeClaimTemplates:
- metadata:
name: test-data
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi
storageClassName: nfs-bogekubectl -n es apply -f test-nginx.yamlkubectl -n es logs quickstart-es-default-0 --all-containers
kubectl -n es logs quickstart-es-default-0 -c elasticsearc
kubectl -n es logs quickstart-es-default-0 -c elasticsearch
kubectl -n es logs quickstart-es-default-0 -c elastic-internal-init-filesystem
kubectl -n es logs quickstart-es-default-0 -c elastic-internal-suspend
kubectl -n es logs quickstart-es-default-0 -c sysctl