多个版本流量区分

发表于 分类于
讲解 Kubernetes 多版本流量区分，演示基于请求头路由，并用 Traefik 与 Istio 实现灰度发布。

服务

默认 svc 和 pod, 将 latest 全局替换成 v1, v2 ，v3 就是三个不同的版本. 给 latest 多创建了一个 svc myapp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-latest
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: myapp-latest
      version: latest
  template:
    metadata:
      labels:
        app: myapp-latest
        version: latest
    spec:
      containers:
      - name: myapp
        image: imwl/myapp:latest
        imagePullPolicy: Always
        ports:
        - name: http
          containerPort: 80

---
apiVersion: v1
kind: Service
metadata:
  name: myapp-latest
  namespace: default
spec:
  selector:
    app: myapp-latest
    version: latest
  ports:
  - name: http
    targetPort: 80
    port: 80

---
apiVersion: v1
kind: Service
metadata:
  name: myapp
  namespace: default
spec:
  selector:
    app: myapp-latest
    version: latest
  ports:
  - name: http
    targetPort: 80
    port: 80

同 namespace 进 pod 测试

1
2
3
4
5
6
7
8
9
10
11
# curl   myapp-latest
Hello MyApp | Version: latest | <a href="hostname.html">Pod Name</a>

# curl   myapp-v1
Hello MyApp | Version: v1 | <a href="hostname.html">Pod Name</a>

# curl   myapp-v2
Hello MyApp | Version: v2 | <a href="hostname.html">Pod Name</a>

# curl   myapp-v3
Hello MyApp | Version: v3 | <a href="hostname.html">Pod Name</a>

使用 header 区分

  1. 新建 nginx 转发, 域名得用完整的内部域名。也可以用变量注入，当前直接用 default

nginx.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
worker_processes 1;

events {
    worker_connections 65536;
}

http {
    resolver kube-dns.kube-system valid=5s; # kube-dns.kube-system 有问题动态注入 pod 里 /etc/resolv.conf 里的 ip。

    server {
        listen 8080;

        location / {
            set_by_lua_block $backend  {
                local version = ngx.req.get_headers()["x-version"]
                if version and version ~= "" then
                    return "myapp-" .. version .. ".default.svc.cluster.local"
                end
                return "myapp.default.svc.cluster.local"
            }

            proxy_pass http://$backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }
    }
}

打包镜像 Dockerfile

1
2
FROM openresty/openresty:alpine-slim
COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf

打包

1
docker buildx build   --platform linux/amd64,linux/arm64/v8  -t imwl/myapp:router   --push .

或者直接使用 cm

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: myapp-router-config
  namespace: default
data:
  nginx.conf: |
    worker_processes 1;

    events {
        worker_connections 65536;
    }

    http {
        resolver kube-dns.kube-system.svc valid=5s;

        server {
            listen 8080;

            location / {
                set_by_lua_block $backend  {
                    local version = ngx.req.get_headers()["x-version"]
                    local namespace = os.getenv("SERVICE_NAMESPACE") or "default"
                    local cluster_domain = os.getenv("CLUSTER_DOMAIN") or "cluster.local"

                    if version and version ~= "" then
                        return "myapp-" .. version .. "." .. namespace .. ".svc." ..  cluster_domain
                    end
                    return "myapp." .. namespace .. ".svc." ..  cluster_domain
                }

                proxy_pass http://$backend;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            }
        }
    }

引用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-router
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: myapp-router
      version: latest
  template:
    metadata:
      labels:
        app: myapp-router
        version: latest
    spec:
      containers:
      - name: myapp-router
        image: openresty/openresty:alpine
        imagePullPolicy: Always
        ports:
        - name: http
          containerPort: 8080
        env:
        - name: SERVICE_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        # 一般都是 cluster.local
        - name: CLUSTER_DOMAIN
          value: cluster.local
        volumeMounts:
          - name: nginx-conf
            mountPath: /usr/local/openresty/nginx/conf/nginx.conf
            subPath: nginx.conf
      volumes:
        - name: nginx-conf
          configMap:
            name: myapp-router-config

---
apiVersion: v1
kind: Service
metadata:
  name: myapp-router
  namespace: default
spec:
  selector:
    app: myapp-router
    version: latest
  ports:
  - name: http
    targetPort: 8080
    port: 80

添加外部访问（非必要）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-router
  namespace: default
  annotations:
    cert-manager.io/cluster-issuer: "letsencrypt-http01"
spec:
  ingressClassName: nginx
  tls:
    - hosts:
      - myapp-router.grafana.eu.org
      secretName: tls-myapp-router.grafana.eu.org
  rules:
    - host: myapp-router.grafana.eu.org
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: myapp-router
                port:
                  number: 80

内部域名测试

1
2
3
4
5
6
7
8
9
10
11
12
myapp-router-78866ffc7b-t4s4k:/# curl myapp-router
Hello MyApp | Version: latest | <a href="hostname.html">Pod Name</a>

myapp-router-78866ffc7b-t4s4k:/# curl -H 'x-version: v1'  myapp-router
Hello MyApp | Version: v1 | <a href="hostname.html">Pod Name</a>

myapp-router-78866ffc7b-t4s4k:/# curl -H 'x-version: v2'  myapp-router
Hello MyApp | Version: v2 | <a href="hostname.html">Pod Name</a>

myapp-router-78866ffc7b-t4s4k:/# curl -H 'x-version: v3'  myapp-router
Hello MyApp | Version: v3 | <a href="hostname.html">Pod Name</a>
myapp-router-78866ffc7b-t4s4k:/#

外部域名测试

1
2
3
4
5
6
7
8
9
10
11
❯ curl -H 'x-version: v1'  myapp-router.grafana.eu.org
Hello MyApp | Version: v1 | <a href="hostname.html">Pod Name</a>

❯ curl -H 'x-version: v2'  myapp-router.grafana.eu.org
Hello MyApp | Version: v2 | <a href="hostname.html">Pod Name</a>

❯ curl -H 'x-version: v3'  myapp-router.grafana.eu.org
Hello MyApp | Version: v3 | <a href="hostname.html">Pod Name</a>

❯ curl   myapp-router.grafana.eu.org
Hello MyApp | Version: latest | <a href="hostname.html">Pod Name</a>

附

myapp 镜像制作

Dockerfile

1
2
3
4
FROM nginx:stable-alpine-slim
COPY nginx.conf /etc/nginx/nginx.conf
COPY default.conf /etc/nginx/conf.d/default.conf
COPY index.html /usr/share/nginx/html/index.html

nginx.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

default.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
server {
    listen       80;
    server_name  localhost;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

    location = /hostname.html {
        alias /etc/hostname;
    }

}

index.html

修改 latest 为 v1， v2, v3 等,然后分别构建

1
Hello MyApp | Version: latest | <a href="hostname.html">Pod Name</a>

构建

1
docker buildx build   --platform linux/amd64,linux/arm64/v8,linux/arm/v7,linux/arm/v6,linux/386,linux/ppc64le,linux/s390x,linux/riscv64  -t imwl/myapp:latest   --push .

traefik 方式

安装

helm 安装

1
2
3
4
5
6
7
8
9
10
helm upgrade --install traefik traefik/traefik \
  --namespace kube-system --create-namespace \
  --set image.registry=swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io \
  --set image.repository=traefik \
  --set image.pullPolicy=IfNotPresent \
  --set ingressRoute.dashboard.enabled=true \
  --set ingressRoute.dashboard.matchRule="PathPrefix(\`/dashboard\`) || PathPrefix(\`/api\`)" \
  --set ingressRoute.dashboard.entryPoints={web} \
  --set providers.kubernetesGateway.enabled=true \
  --set gateway.namespacePolicy=All

使用

  1. 创建证书

ClusterIssuer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-http01-traefik
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: dev@example.com
    privateKeySecretRef:
      name: letsencrypt-http01
    solvers:
    - http01:
        ingress:
          class: traefik
  1. 申请证书
1
2
3
4
5
6
7
8
9
10
11
12
13
14
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: myapp-cert
  namespace: default
spec:
  secretName: tls-myapp-router2.grafana.eu.org
  issuerRef:
    name: letsencrypt-http01-traefik
    kind: ClusterIssuer
  commonName: myapp-router2.grafana.eu.org
  dnsNames:
    - myapp-router2.grafana.eu.org
  1. gateway 使用证书
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: myapp-gw
  namespace: default
spec:
  gatewayClassName: traefik
  listeners:
    - name: https
      protocol: HTTPS
      port: 443
      hostname: myapp-router2.grafana.eu.org
      tls:
        mode: Terminate
        certificateRefs:
          - kind: Secret
            name: tls-myapp-router2.grafana.eu.org
      allowedRoutes:
        namespaces:
          from: Same
  1. ingressroute
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: myapp-router-v2
  namespace: default
spec:
  entryPoints:
    - websecure
  routes:

    - match: Host(`myapp-router2.grafana.eu.org`) && Header(`x-version`, `v1`) && PathPrefix(`/`)
      kind: Rule
      services:
        - name: myapp-v2
          port: 80

    - match: Host(`myapp-router2.grafana.eu.org`) && Header(`x-version`, `v2`) && PathPrefix(`/`)
      kind: Rule
      services:
        - name: myapp-v2
          port: 80

    - match: Host(`myapp-router2.grafana.eu.org`) && Header(`x-version`, `v3`) && PathPrefix(`/`)
      kind: Rule
      services:
        - name: myapp-v2
          port: 80

    # Default fallback route (no header, or other versions)
    - match: Host(`myapp-router2.grafana.eu.org`) && PathPrefix(`/`)
      kind: Rule
      services:
        - name: myapp
          port: 80

  tls:
    secretName: tls-myapp-router2.grafana.eu.org

仅影响外部访问

istio 方式

  1. cert-manager 开启网关申请证书

https://cert-manager.io/docs/configuration/acme/http01/#using-istio-gateway-api

1
2
3
4
5
helm upgrade --install  \
  cert-manager jetstack/cert-manager \
  --namespace common-infra-prod \
  --set installCRDs=true \
  --set "extraArgs={--enable-gateway-api}"
  1. 使用 http01 的方式
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-http01-istio
spec:
  acme:
    email: dev@example.com
    privateKeySecretRef:
      name: letsencrypt-http01-istio
    server: https://acme-v02.api.letsencrypt.org/directory
    solvers:
    - http01:
        gatewayHTTPRoute:
          parentRefs:
            - name: shared-gateway
              namespace: istio-system
              kind: Gateway
  1. 申请证书
1
2
3
4
5
6
7
8
9
10
11
12
13
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: tls-myapp-router3.grafana.eu.org
  namespace: istio-system
spec:
  secretName: tls-myapp-router3.grafana.eu.org
  dnsNames:
    - myapp-router3.grafana.eu.org
  issuerRef:
    name: letsencrypt-http01-istio
    kind: ClusterIssuer
  1. 网关上使用证书（通配符可以使用 *.xxx.xxx http01 方式得一个一个加）
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: shared-gateway
  namespace: istio-system
spec:
  gatewayClassName: istio
  listeners:
    - name: http
      port: 80
      protocol: HTTP
      allowedRoutes:
        namespaces:
          from: All
    - name: myapp-router3.grafana.eu.org
      port: 443
      protocol: HTTPS
      hostname: "myapp-router3.grafana.eu.org"
      tls:
        mode: Terminate
        certificateRefs:
          - name: tls-myapp-router3.grafana.eu.org
            kind: Secret
      allowedRoutes:
        namespaces:
          from: All
  1. httproute 绑定 Gateway(只影响外网访问，内网访问还需要 VirtualService)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: myapp
  namespace: default
spec:
  parentRefs:
    - name: shared-gateway
      namespace: istio-system
  hostnames:
    - myapp-router3.grafana.eu.org
  rules:
    - matches:
        - headers:
            - name: x-version
              value: v1
      backendRefs:
        - name: myapp-v1
          port: 80
    - matches:
        - headers:
            - name: x-version
              value: v2
      backendRefs:
        - name: myapp-v2
          port: 80
    - matches:
        - headers:
            - name: x-version
              value: v3
      backendRefs:
        - name: myapp-v3
          port: 80
    - backendRefs:
        - name: myapp-latest
          port: 80

istio 传统方式，可以一条规则同时影响内外网

安装方式, 会安装 ingressgateway

meshConfig.enableAutoMtls=false 配合 PeerAuthentication 禁用 mTLS 

1
2
3
4
5
istioctl install \
  --set profile=default \
  --set hub=registry.cn-shenzhen.aliyuncs.com/imwl \
  --set meshConfig.enableAutoMtls=false \
  -y
  1. 使用 http01 的方式, 不能使用 gateway
1
2
3
4
5
6
7
8
9
10
11
12
13
14
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-http01-istio
spec:
  acme:
    email: dev@example.com
    server: https://acme-v02.api.letsencrypt.org/directory
    privateKeySecretRef:
      name: letsencrypt-http01-istio
    solvers:
      - http01:
          ingress:
            class: istio
  1. 申请证书
1
2
3
4
5
6
7
8
9
10
11
12
13
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: tls-myapp-router3.grafana.eu.org
  namespace: istio-system
spec:
  secretName: tls-myapp-router3.grafana.eu.org
  dnsNames:
    - myapp-router3.grafana.eu.org
  issuerRef:
    name: letsencrypt-http01-istio
    kind: ClusterIssuer
  1. 网关上使用证书
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: shared-gateway
  namespace: istio-system
spec:
  selector:
    istio: ingressgateway # 这里取决于你的 ingress 网关的 label，通常是这个
  servers:
    - port:
        number: 80
        name: http
        protocol: HTTP
      hosts:
        - "*"
    - port:
        number: 443
        name: https-myapp
        protocol: HTTPS
      hosts:
        - "myapp-router3.grafana.eu.org"
      tls:
        mode: SIMPLE
        credentialName: tls-myapp-router3.grafana.eu.org # 这是 secret 的名称，必须和 cert-manager 生成的一致
  1. VirtualService 绑定网关
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
---
## 添加 可以让内网访问 外网的 https
## 放在 istio-system 全局
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: istio-system
spec:
  mtls:
    mode: PERMISSIVE
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: myapp
  namespace: default
spec:
  exportTo:
    - "*"  # 导出到所有命名空间
  hosts:
    - myapp-router3.grafana.eu.org
    - myapp.default
    - myapp.default.svc.cluster.local
    - myapp.default.svc.cluster.local.
  gateways:
    - mesh # istio 保留，作用于集群内部
    - istio-system/shared-gateway
  http:
    - match:
        - headers:
            x-version:
              exact: v1
      route:
        - destination:
            host: myapp-v1.default.svc.cluster.local
            port:
              number: 80
    - match:
        - headers:
            x-version:
              exact: v2
      route:
        - destination:
            host: myapp-v2.default.svc.cluster.local
            port:
              number: 80
    - match:
        - headers:
            x-version:
              exact: v3
      route:
        - destination:
            host: myapp-v3.default.svc.cluster.local
            port:
              number: 80
    ## 外部禁止访问
    - match:
      - gateways:
        - istio-system/shared-gateway
        uri:
          prefix: /api/internal/
      route:
      - destination:
          host: deny.default.svc.cluster.local
          port:
            number: 80
    ## 内部可以访问
    - match:
      - gateways:
        - mesh
        uri:
          prefix: /api/internal/
      route:
      - destination:
          host: myapp-latest.default.svc.cluster.local
          port:
            number: 80
    # fallback：选一个稳定版本
    - route:
        - destination:
            host: myapp-latest.default.svc.cluster.local
            port:
              number: 80

附，配合流水线

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
#!/bin/bash

set -euo pipefail

CI_COMMIT_REF_NAME_1=${CI_COMMIT_REF_NAME_1:-test}
NAMESPACE="default"
VS_NAME="myapp"
VERSION="${CI_COMMIT_REF_NAME_1}"
TMP_FILE="vs-temp.yaml"
PATCHED_FILE="${TMP_FILE}.patched"
CLEAN_FILE="${TMP_FILE}.clean"

if [[ -z "$VERSION" ]]; then
  echo "ERROR: CI_COMMIT_REF_NAME_1 (VERSION) is empty!"
  exit 1
fi

echo "正在获取现有的 VirtualService: $VS_NAME"
# 获取 VirtualService 到本地文件
kubectl-v1.27.9 get virtualservice "$VS_NAME" -n "$NAMESPACE" -o yaml > "$TMP_FILE"

echo "正在清理重复的版本规则: $VERSION"
# 使用 yq 或 awk 来处理 YAML，先移除所有匹配当前版本的规则
awk -v version="$VERSION" '
BEGIN {
    in_http_block = 0
    in_match_block = 0
    in_version_block = 0
    skip_block = 0
    block_lines = ""
    indent_level = 0
}

# 检测 http: 块的开始
/^  http:/ {
    in_http_block = 1
    print $0
    next
}

# 如果在 http 块中
in_http_block == 1 {
    # 检测新的 match 块开始
    if (/^  - match:/) {
        # 如果之前有积累的块且不需要跳过，则输出
        if (block_lines != "" && skip_block == 0) {
            print block_lines
        }
        # 重置状态
        block_lines = $0
        skip_block = 0
        in_match_block = 1
        next
    }

    # 如果在 match 块中
    if (in_match_block == 1) {
        # 检查是否是我们要删除的版本（精确匹配）
        if (/exact: / && $0 ~ ("exact: " version "$")) {
            skip_block = 1
        }

        # 检测块结束（下一个 - match 或者非缩进行）
        if (/^  - match:/ || /^[^ ]/ || /^spec:/ || /^metadata:/) {
            # 输出当前行之前，先处理积累的块
            if (block_lines != "" && skip_block == 0) {
                print block_lines
            }
            # 重置并开始新块
            if (/^  - match:/) {
                block_lines = $0
                skip_block = 0
                in_match_block = 1
            } else {
                in_http_block = 0
                in_match_block = 0
                print $0
            }
            next
        } else {
            # 积累当前行到块中
            block_lines = block_lines "\n" $0
            next
        }
    }
}

# 非 http 块的行直接输出
in_http_block == 0 {
    print $0
}

END {
    # 处理最后的块
    if (block_lines != "" && skip_block == 0) {
        print block_lines
    }
}
' "$TMP_FILE" > "$CLEAN_FILE"

echo "正在添加新的版本规则: $VERSION"
# 构造要插入的 http 路由段
NEW_ENTRY=$(cat <<EOF
  - match:
    - headers:
        X-Debug-Version:
          exact: ${VERSION}
    route:
    - destination:
        host: myapp-${VERSION}.${NAMESPACE}.svc.cluster.local
        port:
          number: 80
EOF
)

# 在 http: 字段后插入新规则
awk -v insert="$NEW_ENTRY" '
BEGIN { added=0 }
/^  http:/ && added==0 {
    print $0
    print insert
    added=1
    next
}
{ print $0 }
' "$CLEAN_FILE" > "$PATCHED_FILE"

echo "预览更新后的 VirtualService:"
echo "================================"
cat "$PATCHED_FILE"
echo "================================"

echo "正在应用更新到 Kubernetes..."
kubectl-v1.27.9 apply -f "$PATCHED_FILE"

# echo "清理临时文件..."
# rm -f "$TMP_FILE" "$CLEAN_FILE" "$PATCHED_FILE"

echo "VirtualService 更新完成！版本 $VERSION 的规则已添加，重复规则已清理。"