Tips and Trick: Istio and Azure Kubernetes Service (Unmanaged Path)
| Istio Compenets | Managed by |
|---|---|
| istiod | Azure Managed |
| istio-ingressgateway | Azure Managed |
| istio-egressgateway | N/A |
| prometheus | OSS |
| grafana | OSS |
| jaeger | OSS |
Enable istio-ingress-gateway
az aks mesh enable-ingress-gateway --resource-group rg-poc-aks --name poc-aks --ingress-gateway-type external
export INGRESS_HOST_INTERNAL=$(kubectl -n aks-istio-ingress get service aks-istio-ingressgateway-external -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
export INGRESS_PORT_INTERNAL=$(kubectl -n aks-istio-ingress get service aks-istio-ingressgateway-external -o jsonpath='{.spec.ports[?(@.name=="http2")].port}')
export GATEWAY_URL_INTERNAL=$INGRESS_HOST_INTERNAL:$INGRESS_PORT_INTERNAL
echo $GATEWAY_URL_INTERNAL
Dashboard
ControlZ
$ kubectl get pods -n aks-istio-system
NAME READY STATUS RESTARTS AGE
istiod-asm-1-17-6889d9d6c5-mkwsd 1/1 Running 0 131m
istiod-asm-1-17-6889d9d6c5-vqrq8 1/1 Running 0 132m
$ istioctl -i aks-istio-system dashboard controlz istiod-asm-1-17-6889d9d6c5-mkwsd --browser false -n aks-istio-system
http://localhost:9876
Failed to open browser; open http://localhost:9876 in your browser.
Envoy
# istioctl dashboard envoy [<type>/]<name>[.<namespace>] [flags]
$ istioctl -i aks-istio-system dashboard envoy details-v1-7c7dbcb4b5-8v2dh.default
http://localhost:15000
Failed to open browser; open http://localhost:15000 in your browser.
Prometheus
$ wget https://raw.githubusercontent.com/istio/istio/release-1.17/samples/addons/prometheus.yaml -O aks-prometheus.yaml
$ sed -i 's/istio-system/aks-istio-system/g' aks-prometheus.yaml
$ grep istio-system aks-prometheus.yaml
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
$ kubectl apply -f aks-prometheus.yaml
serviceaccount/prometheus created
configmap/prometheus created
clusterrole.rbac.authorization.k8s.io/prometheus unchanged
clusterrolebinding.rbac.authorization.k8s.io/prometheus configured
service/prometheus created
deployment.apps/prometheus created
$ kubectl get svc -n aks-istio-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
istiod-asm-1-17 ClusterIP 10.0.98.209 <none> 15010/TCP,15012/TCP,443/TCP,15014/TCP 10h
prometheus ClusterIP 10.0.219.2 <none> 9090/TCP 43s
$ istioctl -i aks-istio-system dashboard prometheus -n aks-istio-system
http://localhost:9090
Grafana
$ wget https://raw.githubusercontent.com/istio/istio/release-1.17/samples/addons/grafana.yaml -O aks-grafana.yaml
$ sed -i 's/istio-system/aks-istio-system/g' aks-grafana.yaml
$ kubectl apply -f aks-grafana.yaml
serviceaccount/grafana created
configmap/grafana created
service/grafana created
deployment.apps/grafana created
configmap/istio-grafana-dashboards created
configmap/istio-services-grafana-dashboards created
$ istioctl -i aks-istio-system dashboard grafana -n aks-istio-system
http://localhost:3000
``````
```bash title="Jaeger"
$ wget https://raw.githubusercontent.com/istio/istio/release-1.17/samples/addons/jaeger.yaml -O aks-jaeger.yaml
$ sed -i 's/istio-system/aks-istio-system/g' aks-jaeger.yaml
$ kubectl apply -f aks-jaeger.yaml
deployment.apps/jaeger created
service/tracing created
service/zipkin created
service/jaeger-collector created
$ istioctl -i aks-istio-system dashboard jaeger -n aks-istio-system
http://localhost:16686
Kiali
$ wget https://raw.githubusercontent.com/istio/istio/release-1.17/samples/addons/kiali.yaml -O aks-kiali.yaml
$ sed -i 's/istio-system/aks-istio-system/g' aks-kiali.yaml
$ grep istio-system aks-kiali.yaml
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
root_namespace: aks-istio-system
istio_namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
namespace: aks-istio-system
# https://kiali.io/docs/configuration/kialis.kiali.io/
spec:
external_services:
custom_dashboards:
enabled: true
istio:
root_namespace: "aks-istio-system"
config_map_name: "istio-asm-1-17"
istio_api_enabled: true
istiod_deployment_name: "istiod-asm-1-17"
istiod_pod_monitoring_port: 15014
component_status:
enabled: true
components:
- app_label: "istiod"
is_core: true
namespace: aks-istio-system
- app_label: "aks-istio-ingressgateway-external"
is_core: true
namespace: aks-istio-ingress
- app_label: "istio-egressgateway"
is_core: false
namespace: aks-istio-system
prometheus:
health_check_url: http://prometheus.aks-istio-system:9090/-/healthy
url: http://prometheus.aks-istio-system:9090/
tracing:
enabled: true
health_check_url: http://tracing.aks-istio-system:14269/
in_cluster_url: http://tracing.aks-istio-system:16685/
url: http://tracing.aks-istio-system:16685/
grafana:
in_cluster_url: http://grafana.aks-istio-system:3000/
health_check_url: http://grafana.aks-istio-system:3000/api/health
url: http://grafana.aks-istio-system:3000/
enabled: true
dashboards:
- name: "Istio Service Dashboard"
variables:
namespace: "var-namespace"
service: "var-service"
- name: "Istio Workload Dashboard"
variables:
namespace: "var-namespace"
workload: "var-workload"
- name: "Istio Mesh Dashboard"
- name: "Istio Control Plane Dashboard"
- name: "Istio Performance Dashboard"
- name: "Istio Wasm Extension Dashboard"
$ kubectl apply -f aks-kiali.yaml
serviceaccount/kiali created
configmap/kiali created
clusterrole.rbac.authorization.k8s.io/kiali-viewer created
clusterrole.rbac.authorization.k8s.io/kiali created
clusterrolebinding.rbac.authorization.k8s.io/kiali created
role.rbac.authorization.k8s.io/kiali-controlplane created
rolebinding.rbac.authorization.k8s.io/kiali-controlplane created
service/kiali created
deployment.apps/kiali created
$ kubectl -n aks-istio-system rollout restart deployment/kiali
deployment.apps/kiali restarted
$ istioctl -i aks-istio-system dashboard kiali -n aks-istio-system
http://localhost:20001/kiali
Failed to open browser; open http://localhost:20001/kiali in your browser.
Overall Health Check
Istio Version
$ istioctl -i aks-istio-system version
client version: 1.18.1
control plane version: 1.17-dev
data plane version: 1.17.2-distroless (6 proxies)
Verifying connectivity to Istiod
$ wget https://raw.githubusercontent.com/istio/istio/master/samples/sleep/sleep.yaml
$ kubectl create namespace foo
namespace/foo created
$ kubectl apply -f <(istioctl kube-inject -f ./sleep.yaml -i aks-istio-system -r asm-1-17) -n foo
serviceaccount/sleep created
service/sleep created
deployment.apps/sleep created
$ kubectl exec $(kubectl get pod -l app=sleep -n foo -o jsonpath={.items..metadata.name}) -c sleep -n foo -- curl -sS istiod-asm-1-17.aks-istio-system:15014/version
1.17-dev-3e857775086a061d12ee445f32a0b35ea17c8488-Clean
Get an overview of your mesh
# Usage: istioctl proxy-status [<type>/]<name>[.<namespace>] [flags]
$ istioctl -i aks-istio-system proxy-status
NAME CLUSTER CDS LDS EDS RDS ECDS ISTIOD VERSION
details-v1-7c7dbcb4b5-8v2dh.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-mkwsd 1.17.2-distroless
productpage-v1-6c5c9c9d9-plq9r.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-mkwsd 1.17.2-distroless
ratings-v1-844796bf85-tbshm.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-vqrq8 1.17.2-distroless
reviews-v1-5cf854487-6qg5p.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-mkwsd 1.17.2-distroless
reviews-v2-955b74755-gq8r5.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-mkwsd 1.17.2-distroless
reviews-v3-797fc48bc9-svx59.default Kubernetes SYNCED SYNCED SYNCED SYNCED NOT SENT istiod-asm-1-17-6889d9d6c5-mkwsd 1.17.2-distroless
analyze your current live Kubernetes cluster by running
$ istioctl -i aks-istio-system analyze --revision 1-17 --namespace default
✔ No validation issues found when analyzing namespace: default.
$ istioctl -i aks-istio-system analyze --revision 1-17 --all-namespaces
Info [IST0102] (Namespace aks-command) The namespace is not enabled for Istio injection. Run 'kubectl label namespace aks-command istio-injection=enabled' to enable it, or 'kubectl label namespace aks-command istio-injection=disabled' to explicitly mark it as not needing injection.
Info [IST0102] (Namespace aks-istio-ingress) The namespace is not enabled for Istio injection. Run 'kubectl label namespace aks-istio-ingress istio-injection=enabled' to enable it, or 'kubectl label namespace aks-istio-ingress istio-injection=disabled' to explicitly mark it as not needing injection.
Info [IST0102] (Namespace aks-istio-system) The namespace is not enabled for Istio injection. Run 'kubectl label namespace aks-istio-system istio-injection=enabled' to enable it, or 'kubectl label namespace aks-istio-system istio-injection=disabled' to explicitly mark it as not needing injection.
Info [IST0102] (Namespace foo) The namespace is not enabled for Istio injection. Run 'kubectl label namespace foo istio-injection=enabled' to enable it, or 'kubectl label namespace foo istio-injection=disabled' to explicitly mark it as not needing injection.
Analyzing live clusters, local files, or both
$ ls
app-v1.yaml app-v2.yaml gateway.yaml grafana-ab-testing.png README.md virtualservice-match.yaml virtualservice-weight.yaml virtualservice-wildcard.yaml
$ istioctl analyze ./*
Skipping file ./grafana-ab-testing.png, recognized file extensions are: [.json .yaml .yml]
Skipping file ./README.md, recognized file extensions are: [.json .yaml .yml]
✔ No validation issues found when analyzing ./app-v1.yaml
./app-v2.yaml
./gateway.yaml
./virtualservice-match.yaml
./virtualservice-weight.yaml
./virtualservice-wildcard.yaml.
Verify sidecar injection
# Pod
$ istioctl -i aks-istio-system experimental check-inject productpage-v1-6c5c9c9d9-plq9r.default
WEBHOOK REVISION INJECTED REASON
istio-sidecar-injector-asm-1-17-aks-istio-system asm-1-17 ✔ Namespace label istio.io/rev=asm-1-17 matches
# Deployment
$ istioctl -i aks-istio-system experimental check-inject deployment/productpage-v1.default
WEBHOOK REVISION INJECTED REASON
istio-sidecar-injector-asm-1-17-aks-istio-system asm-1-17 ✔ Namespace label istio.io/rev=asm-1-17 matches
Understand your Mesh with Istioctl Describe
$ istioctl -i aks-istio-system experimental describe pod productpage-v1-6c5c9c9d9-plq9r.default
Pod: productpage-v1-6c5c9c9d9-plq9r
Pod Revision: asm-1-17
Pod Ports: 9080 (productpage), 15090 (istio-proxy)
--------------------
Service: productpage
Port: http 9080/HTTP targets pod port 9080
--------------------
Effective PeerAuthentication:
Workload mTLS mode: PERMISSIVE
Skipping Gateway information (no ingress gateway pods)
Deep dive into Envoy configuration
What Envoy version is Istio using?
$ kubectl exec -it productpage-v1-6c5c9c9d9-plq9r -c istio-proxy -n default -- pilot-agent request GET server_info --log_as_json | jq {version}
{
"version": "d799381810ae54f1cccb2a9ae79d9c6191ca2c83/1.25.4-dev/Clean/RELEASE/BoringSSL"
}
Retrieve summary about cluster configuration for a given pod from Envoy.
# istioctl proxy-config clusters <pod-name[.namespace]>
$ istioctl -i aks-istio-system proxy-config cluster productpage-v1-6c5c9c9d9-plq9r.default
SERVICE FQDN PORT SUBSET DIRECTION TYPE DESTINATION RULE
9080 - inbound ORIGINAL_DST
BlackHoleCluster - - - STATIC
InboundPassthroughClusterIpv4 - - - ORIGINAL_DST
PassthroughCluster - - - ORIGINAL_DST
agent - - - STATIC
ama-metrics-ksm.kube-system.svc.cluster.local 8080 - outbound EDS
details.default.svc.cluster.local 9080 - outbound EDS
istiod-asm-1-17.aks-istio-system.svc.cluster.local 443 - outbound EDS
istiod-asm-1-17.aks-istio-system.svc.cluster.local 15010 - outbound EDS
istiod-asm-1-17.aks-istio-system.svc.cluster.local 15012 - outbound EDS
istiod-asm-1-17.aks-istio-system.svc.cluster.local 15014 - outbound EDS
kube-dns.kube-system.svc.cluster.local 53 - outbound EDS
kubernetes.default.svc.cluster.local 443 - outbound EDS
metrics-server.kube-system.svc.cluster.local 443 - outbound EDS
npm-metrics-cluster-service.kube-system.svc.cluster.local 9000 - outbound EDS
productpage.default.svc.cluster.local 9080 - outbound EDS
prometheus_stats - - - STATIC
ratings.default.svc.cluster.local 9080 - outbound EDS
reviews.default.svc.cluster.local 9080 - outbound EDS
sds-grpc - - - STATIC
xds-grpc - - - STATIC
zipkin - - - STRICT_DNS
``````
```bash title="Retrieve summary about route configuration for a given pod from Envoy"
# istioctl proxy-config routes <pod-name[.namespace]>
$ istioctl -i aks-istio-system proxy-config route productpage-v1-6c5c9c9d9-plq9r.default
NAME VHOST NAME DOMAINS MATCH VIRTUAL SERVICE
8080 ama-metrics-ksm.kube-system.svc.cluster.local:8080 ama-metrics-ksm.kube-system, 10.0.21.168 /*
9080 details.default.svc.cluster.local:9080 details, details.default + 1 more... /*
9080 productpage.default.svc.cluster.local:9080 productpage, productpage.default + 1 more... /*
9080 ratings.default.svc.cluster.local:9080 ratings, ratings.default + 1 more... /*
9080 reviews.default.svc.cluster.local:9080 reviews, reviews.default + 1 more... /*
metrics-server.kube-system.svc.cluster.local:443 metrics-server.kube-system.svc.cluster.local:443 * /*
15010 istiod-asm-1-17.aks-istio-system.svc.cluster.local:15010 istiod-asm-1-17.aks-istio-system, 10.0.98.209 /*
15014 istiod-asm-1-17.aks-istio-system.svc.cluster.local:15014 istiod-asm-1-17.aks-istio-system, 10.0.98.209 /*
npm-metrics-cluster-service.kube-system.svc.cluster.local:9000 npm-metrics-cluster-service.kube-system.svc.cluster.local:9000 * /*
InboundPassthroughClusterIpv4 inbound|http|0 * /*
InboundPassthroughClusterIpv4 inbound|http|0 * /*
inbound|9080|| inbound|http|9080 * /*
backend * /healthz/ready*
inbound|9080|| inbound|http|9080 * /*
backend * /stats/prometheus*
``````
```bash title="Retrieve summary about listener configuration for a given pod from Envoy"
$ istioctl -i aks-istio-system proxy-config listeners details-v1-7c7dbcb4b5-8v2dh.default
ADDRESS PORT MATCH DESTINATION
10.0.0.10 53 ALL Cluster: outbound|53||kube-dns.kube-system.svc.cluster.local
10.0.0.1 443 ALL Cluster: outbound|443||kubernetes.default.svc.cluster.local
10.0.154.2 443 Trans: raw_buffer; App: http/1.1,h2c Route: metrics-server.kube-system.svc.cluster.local:443
10.0.154.2 443 ALL Cluster: outbound|443||metrics-server.kube-system.svc.cluster.local
10.0.98.209 443 ALL Cluster: outbound|443||istiod-asm-1-17.aks-istio-system.svc.cluster.local
0.0.0.0 8080 Trans: raw_buffer; App: http/1.1,h2c Route: 8080
0.0.0.0 8080 ALL PassthroughCluster
10.0.34.198 9000 Trans: raw_buffer; App: http/1.1,h2c Route: npm-metrics-cluster-service.kube-system.svc.cluster.local:9000
10.0.34.198 9000 ALL Cluster: outbound|9000||npm-metrics-cluster-service.kube-system.svc.cluster.local
0.0.0.0 9080 Trans: raw_buffer; App: http/1.1,h2c Route: 9080
0.0.0.0 9080 ALL PassthroughCluster
0.0.0.0 15001 ALL PassthroughCluster
0.0.0.0 15001 Addr: *:15001 Non-HTTP/Non-TCP
0.0.0.0 15006 Addr: *:15006 Non-HTTP/Non-TCP
0.0.0.0 15006 Trans: tls; App: istio-http/1.0,istio-http/1.1,istio-h2; Addr: 0.0.0.0/0 InboundPassthroughClusterIpv4
0.0.0.0 15006 Trans: raw_buffer; App: http/1.1,h2c; Addr: 0.0.0.0/0 InboundPassthroughClusterIpv4
0.0.0.0 15006 Trans: tls; App: TCP TLS; Addr: 0.0.0.0/0 InboundPassthroughClusterIpv4
0.0.0.0 15006 Trans: raw_buffer; Addr: 0.0.0.0/0 InboundPassthroughClusterIpv4
0.0.0.0 15006 Trans: tls; Addr: 0.0.0.0/0 InboundPassthroughClusterIpv4
0.0.0.0 15006 Trans: tls; App: istio,istio-peer-exchange,istio-http/1.0,istio-http/1.1,istio-h2; Addr: *:9080 Cluster: inbound|9080||
0.0.0.0 15006 Trans: raw_buffer; Addr: *:9080 Cluster: inbound|9080||
0.0.0.0 15010 Trans: raw_buffer; App: http/1.1,h2c Route: 15010
0.0.0.0 15010 ALL PassthroughCluster
10.0.98.209 15012 ALL Cluster: outbound|15012||istiod-asm-1-17.aks-istio-system.svc.cluster.local
0.0.0.0 15014 Trans: raw_buffer; App: http/1.1,h2c Route: 15014
0.0.0.0 15014 ALL PassthroughCluster
0.0.0.0 15021 ALL Inline Route: /healthz/ready*
0.0.0.0 15090 ALL Inline Route: /stats/prometheus*
Check the listener configuration on a pod
$ istioctl -i aks-istio-system proxy-config listeners details-v1-7c7dbcb4b5-8v2dh.default --port 15001 -o json
[
{
"name": "virtualOutbound",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 15001
}
},
"filterChains": [
{
"filterChainMatch": {
"destinationPort": 15001
},
"filters": [
{
"name": "istio.stats",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/stats.PluginConfig",
"value": {}
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "BlackHoleCluster",
"cluster": "BlackHoleCluster"
}
}
],
"name": "virtualOutbound-blackhole"
},
{
"filters": [
{
"name": "istio.stats",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/stats.PluginConfig",
"value": {}
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "PassthroughCluster",
"cluster": "PassthroughCluster"
}
}
],
"name": "virtualOutbound-catchall-tcp"
}
],
"useOriginalDst": true,
"trafficDirection": "OUTBOUND"
}
]
Retrieve full endpoint configuration for a given pod from Envoy
$ istioctl -i aks-istio-system proxy-config endpoint productpage-v1-6c5c9c9d9-plq9r.default
ENDPOINT STATUS OUTLIER CHECK CLUSTER
10.240.0.5:10091 HEALTHY OK outbound|9000||npm-metrics-cluster-service.kube-system.svc.cluster.local
10.241.0.11:9080 HEALTHY OK outbound|9080||ratings.default.svc.cluster.local
10.241.0.12:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local
10.241.0.14:53 HEALTHY OK outbound|53||kube-dns.kube-system.svc.cluster.local
10.241.0.15:8080 HEALTHY OK outbound|8080||ama-metrics-ksm.kube-system.svc.cluster.local
10.241.0.19:4443 HEALTHY OK outbound|443||metrics-server.kube-system.svc.cluster.local
10.241.0.20:53 HEALTHY OK outbound|53||kube-dns.kube-system.svc.cluster.local
10.241.0.38:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local
10.241.0.40:4443 HEALTHY OK outbound|443||metrics-server.kube-system.svc.cluster.local
10.241.0.43:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local
10.241.0.44:9080 HEALTHY OK outbound|9080||productpage.default.svc.cluster.local
10.241.0.53:9080 HEALTHY OK outbound|9080||details.default.svc.cluster.local
10.241.0.8:15010 HEALTHY OK outbound|15010||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.8:15012 HEALTHY OK outbound|15012||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.8:15014 HEALTHY OK outbound|15014||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.8:15017 HEALTHY OK outbound|443||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.9:15010 HEALTHY OK outbound|15010||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.9:15012 HEALTHY OK outbound|15012||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.9:15014 HEALTHY OK outbound|15014||istiod-asm-1-17.aks-istio-system.svc.cluster.local
10.241.0.9:15017 HEALTHY OK outbound|443||istiod-asm-1-17.aks-istio-system.svc.cluster.local
127.0.0.1:15000 HEALTHY OK prometheus_stats
127.0.0.1:15020 HEALTHY OK agent
20.62.140.71:443 HEALTHY OK outbound|443||kubernetes.default.svc.cluster.local
unix://./etc/istio/proxy/XDS HEALTHY OK xds-grpc
unix://./var/run/secrets/workload-spiffe-uds/socket HEALTHY OK sds-grpc