How To: 設定 Resolve 於 Azure Kubernetes Service

For DNS resolving, you have 2 options:

Pod Level: If you want to resolve the DNS name within a pod.
Cluster Level: If you want to resolve the DNS name within the cluster.

Environment

$ kubectl version --short
Client Version: v1.26.3
Kustomize Version: v4.5.7
Server Version: v1.26.3

$ kubectl cluster-info
Kubernetes control plane is running at https://poc-aks-pwppeq38.hcp.eastus.azmk8s.io:443
CoreDNS is running at https://poc-aks-pwppeq38.hcp.eastus.azmk8s.io:443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
Metrics-server is running at https://poc-aks-pwppeq38.hcp.eastus.azmk8s.io:443/api/v1/namespaces/kube-system/services/https:metrics-server:/proxy

$ kubectl get nodes
NAME                                STATUS   ROLES   AGE   VERSION
aks-nodepool1-36776276-vmss000001   Ready    agent   66m   v1.26.3

Pod Level

1. Use Kubernetes `hostAliases`

Before

without-hostaliases-pod.yml

---
apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: without-hostaliases-pod
spec:
  restartPolicy: Never
  containers:
  - name: cat-hosts
    image: quay.io/tw_pichuang/debug-container:master
    command:
    - "/bin/bash"
    - "-c"
    - "--"
    args:
    - "while true; do sleep 30; done;"

Processes

$ kubectl apply -f without-hostaliases-pod.yml
pod/without-hostaliases-pod created

$ kubectl get pods
NAME                      READY   STATUS    RESTARTS   AGE
without-hostaliases-pod   1/1     Running   0          37s

$ kubectl exec without-hostaliases-pod -- cat /etc/hosts
# Kubernetes-managed hosts file.
127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
fe00::0 ip6-mcastprefix
fe00::1 ip6-allnodes
fe00::2 ip6-allrouters
10.241.0.13     without-hostaliases-pod

$ kubectl exec without-hostaliases-pod -- dig +short blog.pichuang.com.tw
104.21.70.229
172.67.140.43

After

hostaliases-pod.yml

---
apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: hostaliases-pod
spec:
  restartPolicy: Never
  hostAliases:
  - ip: "127.0.0.1"
    hostnames:
    - "blog.pichuang.com.tw"
    - "local.pichuang.com.tw"
  - ip: "10.1.2.3"
    hostnames:
    - "foo.remote"
    - "bar.remote"
  containers:
  - name: cat-hosts
    image: quay.io/tw_pichuang/debug-container:master
    command:
    - "/bin/bash"
    - "-c"
    - "--"
    args:
    - "while true; do sleep 30; done;"

Processes

$ kubectl apply -f hostaliases-pod.yml
pod/hostaliases-pod created

$ kubectl get pods
NAME                      READY   STATUS    RESTARTS   AGE
hostaliases-pod           1/1     Running   0          7s
without-hostaliases-pod   1/1     Running   0          37s

$ kubectl exec hostaliases-pod -- cat /etc/hosts
# Kubernetes-managed hosts file.
127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
fe00::0 ip6-mcastprefix
fe00::1 ip6-allnodes
fe00::2 ip6-allrouters
10.241.0.69     hostaliases-pod

# Entries added by HostAliases.
127.0.0.1       blog.pichuang.com.tw    local.pichuang.com.tw
10.1.2.3        foo.remote      bar.remote

# Get a Shell to a Running Container
$ kubectl exec --stdin --tty hostaliases-pod -- /bin/bash

$ kubectl exec hostaliases-pod -- ping -c3 blog.pichuang.com.tw
PING blog.pichuang.com.tw (127.0.0.1) 56(84) bytes of data.
64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.018 ms
64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.036 ms
64 bytes from localhost (127.0.0.1): icmp_seq=3 ttl=64 time=0.037 ms

--- blog.pichuang.com.tw ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2038ms
rtt min/avg/max/mdev = 0.018/0.030/0.037/0.009 ms

2. Use Kubernetes `dnsConfig`

Pod's DNS Config

Before

without-dnsconfig-pod.yml

---
apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: without-dnsconfig-pod
spec:
  restartPolicy: Never
  containers:
  - name: cat-hosts
    image: quay.io/tw_pichuang/debug-container:master
    command:
    - "/bin/bash"
    - "-c"
    - "--"
    args:
    - "while true; do sleep 30; done;"

Processes

$ kubectl apply -f without-dnsconfig-pod.yml
pod/without-dnsconfig-pod created

$ kubectl exec without-dnsconfig-pod -- cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local zc3wsrpezdhu5hdkc4pchiwajh.bx.internal.cloudapp.net
nameserver 10.0.0.10
options ndots:5

After

dnsconfig-pod.yml

---
apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: dnsconfig-pod
spec:
  restartPolicy: Never
  dnsConfig:
    nameservers:
      - 192.168.78.100 # On-premise DNS Server
    searches:
      - pichuang.local
      - azure.local
    options:
      - name: ndots
        value: "2"
      - name: edns0
  containers:
  - name: cat-hosts
    image: quay.io/tw_pichuang/debug-container:master
    command:
    - "/bin/bash"
    - "-c"
    - "--"
    args:
    - "while true; do sleep 30; done;"

Processes

$ kubectl apply -f dnsconfig-pod.yaml
pod/dnsconfig-pod created

$ kubectl exec dnsconfig-pod -- cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local zc3wsrpezdhu5hdkc4pchiwajh.bx.internal.cloudapp.net pichuang.local azure.local
nameserver 10.0.0.10
nameserver 192.168.78.100
options ndots:2 edns0

Cluster Level

In Azure Kubernetes Service, you should follow the Customize CoreDNS with Azure Kubernetes Service to setup the CoreDNS configuration.

Grab the current resolving configuration

$ kubectl get nodes
NAME                                STATUS   ROLES   AGE     VERSION
aks-nodepool1-36776276-vmss000001   Ready    agent   4h54m   v1.26.3

# Get a Shell to a Running Node
$ kubectl debug node/aks-nodepool1-36776276-vmss000001 -it --image=quay.io/tw_pichuang/debug-container:master
Creating debugging pod node-debugger-aks-nodepool1-36776276-vmss000001-srgx9 with container debugger on node aks-nodepool1-36776276-vmss000001.
If you don't see a command prompt, try pressing enter.

[root@aks-nodepool1-36776276-vmss000001 /]# cat /etc/os-release
NAME="CentOS Stream"
VERSION="8"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="8"
PLATFORM_ID="platform:el8"
PRETTY_NAME="CentOS Stream 8"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:8"
HOME_URL="https://centos.org/"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8"
REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"

[root@aks-nodepool1-36776276-vmss000001 /]# chroot /host /bin/bash

root@aks-nodepool1-36776276-vmss000001:/# cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.2 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.2 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

root@aks-nodepool1-36776276-vmss000001:/# cat  /run/systemd/resolve/resolv.conf
# This is /run/systemd/resolve/resolv.conf managed by man:systemd-resolved(8).
# Do not edit.
#
# This file might be symlinked as /etc/resolv.conf. If you're looking at
# /etc/resolv.conf and seeing this text, you have followed the symlink.
#
# This is a dynamic resolv.conf file for connecting local clients directly to
# all known uplink DNS servers. This file lists all configured search domains.
#
# Third party programs should typically not access this file directly, but only
# through the symlink at /etc/resolv.conf. To manage man:resolv.conf(5) in a
# different way, replace this symlink by a static file or a different symlink.
#
# See man:systemd-resolved.service(8) for details about the supported modes of
# operation for /etc/resolv.conf.

nameserver 168.63.129.16
search zc3wsrpezdhu5hdkc4pchiwajh.bx.internal.cloudapp.net

Show Default Corefile within AKS

kubectl -n kube-system configmaps/coredns

$ kubectl -n kube-system get configmaps coredns -o yaml | kubectl neat -o yaml
apiVersion: v1
data:
  Corefile: |
    .:53 {
        errors
        ready
        health {
          lameduck 5s
        }
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
          ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
        import custom/*.override
    }
    import custom/*.server
kind: ConfigMap
metadata:
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
  name: coredns
  namespace: kube-system

kubectl -n kube-system configmaps/coredns-custom

$ kubectl -n kube-system get configmaps coredns-custom -o yaml | kubectl neat -o yaml
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    addonmanager.kubernetes.io/mode: EnsureExists
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
  name: coredns-custom
  namespace: kube-system

Add Nameserver to custom Corefile

coredns-custom.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns-custom
  namespace: kube-system
data:
  Corefile.override: |
        forward . 8.8.8.8 8.8.4.4

$ kubectl apply -f coredns-custom.yaml
configmap/coredns-custom created

$ kubectl -n kube-system rollout restart deployment coredns
deployment.apps/coredns restarted

$ kubectl -n kube-system get configmaps coredns-custom -o yaml | kubectl neat -o yaml
apiVersion: v1
data:
  Corefile.override: |
    forward . 8.8.8.8 8.8.4.4
kind: ConfigMap
metadata:
  labels:
    addonmanager.kubernetes.io/mode: EnsureExists
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
  name: coredns-custom
  namespace: kube-system

How To: 設定 Resolve 於 Azure Kubernetes Service

Environment

Pod Level

1. Use Kubernetes hostAliases

Before

After

2. Use Kubernetes dnsConfig

Before

After

Cluster Level

Grab the current resolving configuration

Show Default Corefile within AKS

Add Nameserver to custom Corefile

References

1. Use Kubernetes `hostAliases`

2. Use Kubernetes `dnsConfig`