지금까지 구축된 kubernetes에서 ingress-nginx를 설치하고 "Deployment, Service, HorizontalPodAutoscaler"로 구성된 앱을 배포하고
ingress를 구성 후, 최종 서비스 형태를 만드는것을 목표로 한다.
ingress-nginx를 구축하여 서비스하는 구성도
서비스용 앱 배포
모든 요청에 응답하는 echo server를 앱으로 아래와 같은 파일로 구성해서 배포한다.
echo-server-dpm.yaml - Deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: echo-server-dpm
labels:
app: echo-server
spec:
replicas: 1
selector:
matchLabels:
app: echo-server
template:
metadata:
labels:
app: echo-server
spec:
containers:
- name: echo-server-pod
image: ealen/echo-server
resources:
requests:
cpu: 100m
memory: 200Mi
ports:
- name: http
containerPort: 80# kubectl apply -f echo-server-dpm.yaml
echo-server-svc.yaml - Service
---
apiVersion: v1
kind: Service
metadata:
name: echo-server-svc
labels:
app: echo-server
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
selector:
app: echo-server# kubectl apply -f echo-server-svc.yaml
echo-server-hpa.yaml - HorizontalPodAutoscaler
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: echo-server-hpa
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: echo-server-dpm
minReplicas: 2
maxReplicas: 100
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 30# kubectl apply -f echo-server-hpa.yaml
구성 검증
1. HorizontalPodAutoscaler
# kubectl get hpa
NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE
echo-server-hpa Deployment/echo-server-dpm 0%/30% 2 100 2 15m위와 같이 배포된 hpa에서 메트릭이 올바로 수집되고 있는지 확인한다.
2. Service
# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
echo-server-svc ClusterIP 10.233.28.255 <none> 80/TCP 16m위와 같이 svc가 구성되어 있는지 확인한다.
3. 서비스 확인 (echo server pod에 연결해서 localhost/hello, echo-server-svc/hello로 호출해본다.
root@homemachine:~/echo-server-deployment# kubectl get pod
NAME READY STATUS RESTARTS AGE
echo-server-dpm-5f6899f47d-9l954 1/1 Running 0 20m
echo-server-dpm-5f6899f47d-tz64q 1/1 Running 0 17m
metrics-server-55cc5bcdb8-qfkhv 1/1 Running 0 21h
root@homemachine:~/echo-server-deployment# kubectl exec -ti echo-server-dpm-5f6899f47d-9l954 -- /bin/sh
/app # wget -O - localhost/hello
Connecting to localhost ([::1]:80)
writing to stdout
{"host":{"hostname":"localhost","ip":"::1","ips":[]},"http":{"method":"GET","baseUrl":"","originalUrl":"/hello","protocol":"http"},"request":{"params":{"0":"/hello"},"query":{},"cookies":{},"body":{},"headers":{"host":"localhost","user-agent":"Wget","connection":"close"}},"environment":{"PATH":"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","HOSTNAME":"echo-server-dpm-5f6899f47d-9l954","NODE_VERSION":"20.11.0","YARN_VERSION":"1.22.19","METRICS_SERVER_SERVICE_HOST":"10.233.20.91","METRICS_SERVER_PORT":"tcp://10.233.20.91:443","KUBERNETES_SERVICE_HOST":"10.233.0.1","KUBERNETES_SERVICE_PORT":"443","KUBERNETES_SERVICE_PORT_HTTPS":"443","KUBERNETES_PORT_443_TCP_PORT":"443","KUBERNETES_PORT_443_TCP_ADDR":"10.233.0.1","METRICS_SERVER_PORT_443_TCP":"tcp://10.233.20.91:443","METRICS_SERVER_PORT_443_TCP_PORT":"443","KUBERNETES_PORT_443_TCP_PROTO":"tcp","METRICS_SERVER_SERVICE_PORT_HTTPS":"443","METRICS_SERVER_PORT_443_TCP_ADDR":"10.233.20.91","KUBERNETES_PORT":"tcp://10.233.0.1:443","KUBERNETES_PORT_443_TCP":"tcp://10.233.0.1:443","METRICS_SERVER_SERVICE_PORT":"443","METRICS_SERVER_PORT_443_TCP_PROTO":"- 100% |*****************************************************************************| 1146 0:00:00 ETA
written to stdout
/app # wget -O - echo-server-svc/hello
Connecting to echo-server-svc (10.233.28.255:80)
writing to stdout
{"host":{"hostname":"echo-server-svc","ip":"::ffff:10.233.74.70","ips":[]},"http":{"method":"GET","baseUrl":"","originalUrl":"/hello","protocol":"http"},"request":{"params":{"0":"/hello"},"query":{},"cookies":{},"body":{},"headers":{"host":"echo-server-svc","user-agent":"Wget","connection":"close"}},"environment":{"PATH":"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","HOSTNAME":"echo-server-dpm-5f6899f47d-tz64q","NODE_VERSION":"20.11.0","YARN_VERSION":"1.22.19","ECHO_SERVER_SVC_PORT":"tcp://10.233.28.255:80","ECHO_SERVER_SVC_PORT_80_TCP_PROTO":"tcp","METRICS_SERVER_SERVICE_HOST":"10.233.20.91","METRICS_SERVER_PORT_443_TCP_PORT":"443","METRICS_SERVER_PORT_443_TCP_ADDR":"10.233.20.91","KUBERNETES_SERVICE_HOST":"10.233.0.1","ECHO_SERVER_SVC_SERVICE_PORT_HTTP":"80","METRICS_SERVER_SERVICE_PORT":"443","METRICS_SERVER_SERVICE_PORT_HTTPS":"443","KUBERNETES_PORT":"tcp://10.233.0.1:443","ECHO_SERVER_SVC_PORT_80_TCP_PORT":"80","ECHO_SERVER_SVC_PORT_80_TCP_ADDR":"10.233.28.255","ECHO_SERVER_SVC_SERVICE_PORT":"80","KUBERNETES_SERVICE_PORT_HTTPS":"443","KUBERNETES_PORT_443_TCP":"tcp://10.233.0.1:443","KUBERNETES_PORT_443_TCP_PROTO":"tcp","KUBERNETES_PORT_443_TCP_ADDR":"10.233.0.1","ECHO_SERVER_SVC_SERVICE_HOST":"10.233.28.255","ECHO_SERVER_SVC_PORT_80_TCP":"tcp://10.233.28.255:80","METRICS_SERVER_PORT":"tcp://10.233.20.91:443","METRICS_SERVER_PORT_443_TCP":"tcp://10.233.20.91:443","METRICS_SERVER_PORT_443_TCP_PROTO":"tcp","KUBERNETES_SERVICE_PORT":"443","KUBERNETES_PORT_44- 100% |*****************************************************************************| 1534 0:00:00 ETA
written to stdout위와 같이 정상적인 호출이 온다면 앱은 정상적으로 배포되었다.
ingress-nginx 설치
아래와 같이 helm으로 template(ingress-nginx-deploy.yaml)을 생성한다.
차후 prometheus-operator를 구성해서 ingress-nginx의 metric 수집을 위한 옵션을 추가했고, ingress-nginx의 autoscale을 위해 옵션을 추가했다.
helm template ingress-nginx ingress-nginx \
--repo https://kubernetes.github.io/ingress-nginx \
--dependency-update --include-crds \
--set controller.metrics.enabled=true \
--set-string controller.podAnnotations."prometheus\.io/scrape"="true" \
--set-string controller.podAnnotations."prometheus\.io/port"="10254" \
--set controller.autoscaling.enabled=true \
--set controller.autoscaling.minReplicas=1 \
--set controller.autoscaling.maxReplicas=100 \
--set controller.autoscaling.targetCPUUtilizationPercentage=30 > ingress-nginx-deploy.yaml# kubectl apply -f ingress-nginx-deploy.yaml --server-side추가로 "--server-side" 옵션으로 배포를 추천한다.
https://kubernetes.github.io/ingress-nginx/deploy/
https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/monitoring.md
구성된 service를 확인해보면 아래와 같이 "192.168.122.11" IP로 Loadbalancer가 구성되어 있는 것을 확인할 수 있다. (상황에 따라 다른 IP가 할당될 수 있다.)
# kubectl get svc|grep ingress
ingress-nginx-controller LoadBalancer 10.233.39.240 192.168.122.11 80:31917/TCP,443:30506/TCP 79s
ingress-nginx-controller-admission ClusterIP 10.233.52.85 <none> 443/TCP 79s
ingress-nginx-controller-metrics ClusterIP 10.233.23.2 <none> 10254/TCP 79s
"192.168.122.11" IP로 호출해보면 ingress가 등록되지 않아 404 에러가 리턴된다.
# curl 192.168.122.11
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx</center>
</body>
</html>
서비스용 앱 Ingress 배포
echo-server-ing.yaml - Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: echo-server-ing
spec:
ingressClassName: nginx
rules:
- host:
http:
paths:
- pathType: Prefix
path: /hello
backend:
service:
name: echo-server-svc
port:
number: 80# kubectl apply -f echo-server-ing.yaml
Loadbalancer(192.168.122.11)로 호출 테스트를 진행해보면 아래와 같이 결과가 나와야 정상적으로 구축된 것이다.
# curl 192.168.122.11/hello
{"host":{"hostname":"192.168.122.11","ip":"::ffff:10.233.71.7","ips":[]},"http":{"method":"GET","baseUrl":"","originalUrl":"/hello","protocol":"http"},"request":{"params":{"0":"/hello"},"query":{},"cookies":{},"body":{},"headers":{"host":"192.168.122.11","x-request-id":"771b6bf061a0b118e8d912fcdeddfda6","x-real-ip":"10.233.120.0","x-forwarded-for":"10.233.120.0","x-forwarded-host":"192.168.122.11","x-forwarded-port":"80","x-forwarded-proto":"http","x-forwarded-scheme":"http","x-scheme":"http","user-agent":"curl/7.81.0","accept":"*/*"}},"environment":{"PATH":"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","HOSTNAME":"echo-server-dpm-5f6899f47d-tz64q","NODE_VERSION":"20.11.0","YARN_VERSION":"1.22.19","ECHO_SERVER_SVC_PORT":"tcp://10.233.28.255:80","ECHO_SERVER_SVC_PORT_80_TCP_PROTO":"tcp","METRICS_SERVER_SERVICE_HOST":"10.233.20.91","METRICS_SERVER_PORT_443_TCP_PORT":"443","METRICS_SERVER_PORT_443_TCP_ADDR":"10.233.20.91","KUBERNETES_SERVICE_HOST":"10.233.0.1","ECHO_SERVER_SVC_SERVICE_PORT_HTTP":"80","METRICS_SERVER_SERVICE_PORT":"443","METRICS_SERVER_SERVICE_PORT_HTTPS":"443","KUBERNETES_PORT":"tcp://10.233.0.1:443","ECHO_SERVER_SVC_PORT_80_TCP_PORT":"80","ECHO_SERVER_SVC_PORT_80_TCP_ADDR":"10.233.28.255","ECHO_SERVER_SVC_SERVICE_PORT":"80","KUBERNETES_SERVICE_PORT_HTTPS":"443","KUBERNETES_PORT_443_TCP":"tcp://10.233.0.1:443","KUBERNETES_PORT_443_TCP_PROTO":"tcp","KUBERNETES_PORT_443_TCP_ADDR":"10.233.0.1","ECHO_SERVER_SVC_SERVICE_HOST":"10.233.28.255","ECHO_SERVER_SVC_PORT_80_TCP":"tcp://10.233.28.255:80","METRICS_SERVER_PORT":"tcp://10.233.20.91:443","METRICS_SERVER_PORT_443_TCP":"tcp://10.233.20.91:443","METRICS_SERVER_PORT_443_TCP_PROTO":"tcp","KUBERNETES_SERVICE_PORT":"443","KUBERNETES_PORT_443_TCP_PORT":"443","HOME":"/root"}}