'when EKS ConfigMap meet AWS Secret manager

Here I will demo a mysql as database and a wordpress deployment as backend in EKS to reference ConfigMap and Secret via environment variables during run time by reading the environment-specific configuration (like environment names or feature flags) from a ConfigMap and sensitive information (like a database password) from a Secret.

- Create demo resources

# vim cf-demo.yaml to create demo namespace,  configmap, secret and deployment

root@asb:~/cf# cat secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: mysql-root-secret
type: Opaque
data:
  MYSQL_ROOT_PASSWORD: c2VjdXJlcGFzc3dvcmQ=  # Base64 for "securepassword"

root@asb:~/cf# cat mysql.yaml 
apiVersion: v1
kind: Service
metadata:
  name: mysql-service
spec:
  ports:
    - port: 3306
  selector:
    app: mysql
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql
spec:
  selector:
    matchLabels:
      app: mysql
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - name: mysql
        image: mysql:5.7
        env:
        - name: MYSQL_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mysql-root-secret
              key: MYSQL_ROOT_PASSWORD

root@asb:~/cf# cat wordpress.yaml 

apiVersion: v1
kind: ConfigMap
metadata:
  name: wordpress-config
data:
  WORDPRESS_DB_HOST: "mysql-service:3306"
  WORDPRESS_DB_NAME: "wordpress"
---
apiVersion: v1
kind: Secret
metadata:
  name: wordpress-secret
type: Opaque
data:
  WORDPRESS_DB_USER: d29yZHByZXNz  # Base64 for "wordpress"
  WORDPRESS_DB_PASSWORD: c2VjdXJlcGFzc3dvcmQ=  # Base64 for "securepassword"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: wordpress
spec:
  replicas: 1
  selector:
    matchLabels:
      app: wordpress
  template:
    metadata:
      labels:
        app: wordpress
    spec:
      containers:
      - name: wordpress
        image: wordpress:latest
        ports:
        - containerPort: 80
        env:
        - name: WORDPRESS_DB_HOST
          valueFrom:
            configMapKeyRef:
              name: wordpress-config
              key: WORDPRESS_DB_HOST
        - name: WORDPRESS_DB_NAME
          valueFrom:
            configMapKeyRef:
              name: wordpress-config
              key: WORDPRESS_DB_NAME
        - name: WORDPRESS_DB_USER
          valueFrom:
            secretKeyRef:
              name: wordpress-secret
              key: WORDPRESS_DB_USER
        - name: WORDPRESS_DB_PASSWORD
          valueFrom:
            secretKeyRef:
              name: wordpress-secret
              key: WORDPRESS_DB_PASSWORD
---
apiVersion: v1
kind: Service
metadata:
  name: wordpress-service
spec:
  selector:
    app: wordpress
  ports:
    - protocol: TCP
      port: 80
      targetPort: 80
  type: LoadBalancer  # Or NodePort if LoadBalancer is not available

- Apply and verify the Configuration and Secrets in the Pod

root@asb:~/cf# kubectl create ns cf-test
root@asb:~/cf# kubectl apply -f . -n cf-test

configmap/wordpress-config created
secret/wordpress-secret created
deployment.apps/wordpress created
service/wordpress-service created
secret/mysql-root-secret created
service/mysql-service created
deployment.apps/mysql created

root@asb:~/cf# kubectl get all -n cf-test 
NAME                             READY   STATUS    RESTARTS   AGE
pod/mysql-fdff667f8-xzblb        1/1     Running   0          6s
pod/wordpress-6dff4575b9-sgn8t   1/1     Running   0          12m

NAME                        TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
service/mysql-service       ClusterIP      10.106.191.27           3306/TCP       6s
service/wordpress-service   LoadBalancer   10.97.169.213   pending       80:31208/TCP   12m

NAME                        READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/mysql       1/1     1            1           6s
deployment.apps/wordpress   1/1     1            1           12m

NAME                                   DESIRED   CURRENT   READY   AGE
replicaset.apps/mysql-fdff667f8        1         1         1       6s
replicaset.apps/wordpress-6dff4575b9   1         1         1       12m

# Kubectl exec into pod to verify the configmap and secret

root@asb:~/cf# kubectl logs wordpress-6dff4575b9-sgn8t -n cf-test 
WordPress not found in /var/www/html - copying now...
Complete! WordPress has been successfully copied to /var/www/html
No 'wp-config.php' found in /var/www/html, but 'WORDPRESS_...' variables supplied; copying 'wp-config-docker.php' (WORDPRESS_DB_HOST WORDPRESS_DB_NAME WORDPRESS_DB_PASSWORD WORDPRESS_DB_USER WORDPRESS_SERVICE_PORT WORDPRESS_SERVICE_PORT_80_TCP WORDPRESS_SERVICE_PORT_80_TCP_ADDR WORDPRESS_SERVICE_PORT_80_TCP_PORT WORDPRESS_SERVICE_PORT_80_TCP_PROTO WORDPRESS_SERVICE_SERVICE_HOST WORDPRESS_SERVICE_SERVICE_PORT)
AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 192.168.48.245. Set the 'ServerName' directive globally to suppress this message
AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 192.168.48.245. Set the 'ServerName' directive globally to suppress this message
[Wed Nov 06 23:39:29.456836 2024] [mpm_prefork:notice] [pid 1:tid 1] AH00163: Apache/2.4.62 (Debian) PHP/8.2.25 configured -- resuming normal operations
[Wed Nov 06 23:39:29.456923 2024] [core:notice] [pid 1:tid 1] AH00094: Command line: 'apache2 -D FOREGROUND'

root@asb:~/cf# kubectl exec -it wordpress-6dff4575b9-sgn8t -n cf-test -- env | grep WORDPRESS_DB
WORDPRESS_DB_NAME=wordpress
WORDPRESS_DB_USER=wordpress
WORDPRESS_DB_PASSWORD=securepassword
WORDPRESS_DB_HOST=mysql-service:3306

IRSA with AWS Secret Manager to ensure security best practices in EKS

Some best practices for using secrets in AWS EKS include:

• Avoid Storing Secrets in Kubernetes: Instead of storing sensitive information in Kubernetes Secrets, use AWS Secrets Manager to store secrets securely.
• Use IAM Roles for Authorization: Rely on IAM roles and policies to control access to secrets in AWS Secrets Manager, ensuring that only authorized pods can retrieve specific secrets.
• Retrieve Secrets at Runtime: Configure your application to retrieve secrets directly from AWS Secrets Manager at runtime, using an SDK or API, rather than storing secrets as environment variables.
• Automate Secret Rotation: Set up AWS Secrets Manager to rotate secrets automatically, reducing the risk of stale or compromised credentials.

Here I will run a Step-by-Step Guide to Using AWS Secrets Manager with Kubernetes in EKS so the application can retrieve secrets at Runtime from AWS Secret Manager with IRSA (Identity and Access Management Roles for Service Accounts).

- set up AWS Secrets Manager and create a secret, then create a IAM policy attach to IAM role, use EKS OIDC provider together with namespace and service account, so our deployment with an IAM role will have access to the secret in AWS.

# Create a new AWS Secrets Manager secret
aws secretsmanager create-secret \
 --name prod/myapp/db \
 --description "Secret for database credentials" \
 --secret-string '{"DB_USER": "mydatabaseuser", "DB_PASSWORD": "mypassword"}'

# create an IAM policy named ReadProdMyAppDBSecret with read-only access to the prod/myapp/db secret
aws iam create-policy \
 --policy-name ReadProdMyAppDBSecret \
 --policy-document '{
 "Version": "2012-10-17",
 "Statement": [
 {
 "Effect": "Allow",
 "Action": "secretsmanager:GetSecretValue",
 "Resource": "arn:aws:secretsmanager:$REGION:$ACCOUNT_ID:secret:prod/myapp/db*"
 }
 ]
 }'

# get EKS OIDC provider  
aws eks describe-cluster --name zack-eks-cluster --query "cluster.identity.oidc.issuer" --output text

# create the trust policy file, use EKS OIDC provider together with namespace and service account
# vim trust-policy.json
{
 "Version": "2012-10-17",
 "Statement": [
 {
 "Effect": "Allow",
 "Principal": {
 "Federated": "arn:aws:iam::$ACCOUNT_ID:oidc-provider/oidc.eks.$REGION.amazonaws.com/id/zack-eks-cluster"
},
 "Action": "sts:AssumeRoleWithWebIdentity",
 "Condition": {
 "StringEquals": {
 "arn:aws:iam::$ACCOUNT_ID:oidc-provider/oidc.eks.$REGION.amazonaws.com/id/zack-eks-cluster:sub": "system:serviceaccount:ns-zack-irsa-demo:sa-zack-irsa-demo"
 }
 }
 }
 ]
}

# create the IAM role with the trust policy
aws iam create-role \
 --role-name EKSSecretsAccessRole \
 --assume-role-policy-document file://trust-policy.json

# Attach the Policy to the Role
aws iam attach-role-policy \
 --role-name EKSSecretsAccessRole \
 --policy-arn arn:aws:iam:#ACCOUNT_ID:policy/ReadProdMyAppDBSecret

- Create namespace and service account in EKS and annotate it with above IAM role ARN

# create service account "sa-zack-irsa-demo" in namespace "ns-zack-irsa-demo" with annotation of IAM role ARN:

apiVersion: v1
kind: ServiceAccount
metadata:
 name: sa-zack-irsa-demo
 namespace: ns-zack-irsa-demo
 annotations:
 eks.amazonaws.com/role-arn: arn:aws:iam::$ACCOUNT_ID:role/EKSSecretsAccessRole

- Create Zackblog Deployment to use the service account created above, so that pods launched by the Deployment will inherit permissions to access the secret

apiVersion: apps/v1
kind: Deployment
metadata:
 name: zackblog-app
 namespace: ns-zack-irsa-demo
spec:
 replicas: 1
 selector:
 matchLabels:
 app: zackblog-app
 template:
 metadata:
 labels:
 app: zackblog-app
 spec:
 serviceAccountName: sa-zack-irsa-demo  # Use the custom service account
 containers:
 - name: zackblog-app
   image: zackz001/gitops-jekyll:latest

- Modify Application Code to Retrieve Secrets at Runtime, bellow python with boto3 can retrieve the `DB_USER` and `DB_PASSWORD` fields from the secret in AWS Secrets Manager

import boto3
import json

def get_secret():
 secret_name = "prod/myapp/db"
 region_name = "ap-southeast-2"

 client = boto3.client("secretsmanager", region_name=region_name)

 response = client.get_secret_value(SecretId=secret_name)
 secret = json.loads(response["SecretString"])

 db_user = secret["DB_USER"]
 db_password = secret["DB_PASSWORD"]

 return db_user, db_password

Conclusion

This setup with EKS and IAM role with Service Account (IRSA), which combines AWS Secrets Manager, IAM roles, and Kubernetes Service Accounts, provides a secure and scalable way to manage sensitive information in Kubernetes on EKS.

More to be considered to ensure EKS security best practices:

• Enable Encryption: AWS Secrets Manager secrets are encrypted by default
• Limit IAM Permissions: Follow the principle of least privilege by restricting IAM permissions to only the required secrets.
• Audit and Monitor Access: Use AWS CloudTrail to monitor access to secrets for security and auditing purposes.