Tình huống là như thế này:
Bạn có 1 workload(deployment, statefulSet, Job) trên k8s
Bạn muốn là workload trên k8s cũng có thể access các resources trên AWS thì làm sao?
Cách 1: add access key và secret key vào workload thông qua Secret và Environment.
Cách 2: sử dụng Service Account đã được cấp quyền access và gắn vào workload(ở bài này thì mình dùng cách 2)
1) Introduction to IRSA
2) Practice configuring IRSA to access S3, resources on AWS.
Extract OIDC provider thumbprint for all AWS Region with EKS support
oidc-thumbprint-regions.sh
#!/bin/bash
set -e
if [ ! -z "$DEBUG" ] ; then
set -x
fi
REGIONS="us-east-2
us-east-1
us-west-1
us-west-2
ap-east-1
ap-northeast-2
ap-southeast-1
ap-southeast-2
ap-northeast-1
eu-central-1
eu-west-1
eu-west-2
eu-west-3
eu-north-1
me-south-1
sa-east-1"
for REGION in $REGIONS ; do
JWKS_URI="oidc.eks.${REGION}.amazonaws.com"
# Extract all certificates in separate files
# https://unix.stackexchange.com/questions/368123/how-to-extract-the-root-ca-and-subordinate-ca-from-a-certificate-chain-in-linux
TEMP=$(mktemp -d -t oidc-eks-XXXX)
openssl s_client -servername $JWKS_URI -showcerts -connect $JWKS_URI:443 < /dev/null 2>/dev/null | awk -v dir="$TEMP" '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/{ if(/BEGIN/){a++}; out=dir"/cert00"a".crt"; print >out }'
# Assume last found certificate in chain is the ROOT_CA
ROOT_CA=$(ls -1 $TEMP/* | tail -1)
# Extract fingerprint in desired format (no header, no colons)
THUMBPRINT=$(openssl x509 -fingerprint -noout -in $ROOT_CA | sed 's/^.*=//' | sed 's/://g')
printf '{"%s": "%s"}\n' $REGION $THUMBPRINT
rm -rf $TEMP
done
Kiểm tra OIDC provider thumbprint khi nào thì hết hạn
# Enable IAM Roles for EKS Service-Accounts (IRSA). # The Root CA Thumbprint for an OpenID Connect Identity Provider is currently # Being passed as a default value which is the same for all regions and # Is valid until (Jun 28 17:39:16 2034 GMT). # https://crt.sh/?q=9E99A48A9960B14926BB7F3B02E22DA2B0AB7280 # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html # https://github.com/terraform-providers/terraform-provider-aws/issues/10104
Bên terraform của cluster EKS mình sẽ tạo ra 2 file nữa
c6-01-iam-oidc-connect-provider-variables.tf
>>>>>>>>>>
# Input Variables - AWS IAM OIDC Connect Provider
# EKS OIDC ROOT CA Thumbprint - valid until 2037
variable "eks_oidc_root_ca_thumbprint" {
type = string
description = "Thumbprint of Root CA for EKS OIDC, Valid until 2037"
default = "9e99a48a9960b14926bb7f3b02e22da2b0ab7280"
}
c6-02-iam-oidc-connect-provider.tf
>>>>>>
>>>>>
# Datasource: AWS Partition
# Use this data source to lookup information about the current AWS partition in which Terraform is working
data "aws_partition" "current" {}
# Resource: AWS IAM Open ID Connect Provider
resource "aws_iam_openid_connect_provider" "oidc_provider" {
client_id_list = ["sts.${data.aws_partition.current.dns_suffix}"]
thumbprint_list = [var.eks_oidc_root_ca_thumbprint]
url = aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer
tags = merge(
{
Name = "${var.cluster_name}-eks-irsa"
},
local.common_tags
)
}
# Output: AWS IAM Open ID Connect Provider ARN
output "aws_iam_openid_connect_provider_arn" {
description = "AWS IAM Open ID Connect Provider ARN"
value = aws_iam_openid_connect_provider.oidc_provider.arn
}
# Extract OIDC Provider from OIDC Provider ARN
locals {
aws_iam_oidc_connect_provider_extract_from_arn = element(split("oidc-provider/", "${aws_iam_openid_connect_provider.oidc_provider.arn}"), 1)
}
# Output: AWS IAM Open ID Connect Provider
output "aws_iam_openid_connect_provider_extract_from_arn" {
description = "AWS IAM Open ID Connect Provider extract from ARN"
value = local.aws_iam_oidc_connect_provider_extract_from_arn
}
# Sample Outputs for Reference
/*
aws_iam_openid_connect_provider_arn = "arn:aws:iam::180789647333:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/A9DED4A4FA341C2A5D985A260650F232"
aws_iam_openid_connect_provider_extract_from_arn = "oidc.eks.us-east-1.amazonaws.com/id/A9DED4A4FA341C2A5D985A260650F232"
*/
Manifest đầy đủ ở đây:
https://github.com/mrnim94/terraform-aws/tree/master/eks/AWS-EKS-Cluster-Basics
Chúng ta có 1 resource
khi run terraform apply thì: aws_iam_openid_connect_provider
==> tạo 1 IAM access vào nhà cung cấp OpenID
trong resource trên có 1 field: url = aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer
thì nó lấy ở đây:
Làm sao để lấy được cài OpenID connect Provider URL thông qua terraform thì tham khảo ở đây:
https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster#enabling-iam-roles-for-service-accounts
Chúng ta đã tạo được
Còn cái: client_id_list = [“sts.${data.aws_partition.current.dns_suffix}”]
thường nó sẽ là client_id_list = [“sts.amazonaws.com”]
Hoặc bạn lấy động như mình
tham khảo link: dns_suffix – Base DNS domain name for the current partition (e.g. amazonaws.com in AWS Commercial, amazonaws.com.cn in AWS China).
Ở đâu mình có sử dụng merge map
tags = merge(
{
Name = "${var.cluster_name}-eks-irsa"
},
local.common_tags
)
Tham khảo link này: https://www.terraform.io/language/functions/merge
Ngoài ra mình có sử dụng spit function
https://www.terraform.io/language/functions/split
EKS OpenID Connect Well Known Configuration URL
- We can also call it as
OpenID Connect Discovery URL - Discovery: Defines how Clients dynamically discover information about OpenID Providers
# Get OpenID Connect provider URL for EKS Cluster Go to Services -> EKS -> hr-dev-eksdemo1 -> Configuration -> Details -> OpenID Connect provider URL # EKS OpenID Connect Well Known Configuration URL <EKS OpenID Connect provider URL>/.well-known/openid-configuration # Sample https://oidc.eks.us-east-1.amazonaws.com/id/EC973221A6C1BC248C79CFD5455EEECC/.well-known/openid-configuration
Acquiring state lock. This may take a few moments... ╷ │ Error: AccessDenied: Access Denied │ status code: 403, request id: BK8EFDSKVEYMW37W, host id: VIfTThsTvogmR4nLHcvBsEfbKnNQ7ecVpfxftkgjn4jFU5P5vpT+n5O71LM+gsfrv9xbW3X+kOg= │ │ ╵ Releasing state lock. This may take a few moments...
Bạn nên tìm đến terraform_remote_state và sửa config bucket
Bạn tạo role để mấy thanh niên mà kết nối đến oidc của EKS và có service account là irsa-demo-sa
Dùng terraform tạo 1 service account trong k8s
c4-03-irsa-k8s-service-account.tf
>>>>
>>
# Resource: Kubernetes Service Account
resource "kubernetes_service_account_v1" "irsa_demo_sa" {
depends_on = [ aws_iam_role_policy_attachment.irsa_iam_role_policy_attach ]
metadata {
name = "irsa-demo-sa"
annotations = {
"eks.amazonaws.com/role-arn" = aws_iam_role.irsa_iam_role.arn
}
}
}
Đây là Service account mà get được trong k8s
root@LP11-D7891:~# kubectl get sa/irsa-demo-sa -o yaml
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
annotations:
eks.amazonaws.com/role-arn: arn:aws:iam::250887682577:role/SAP-dev-irsa-iam-role
creationTimestamp: "2022-09-08T05:39:20Z"
name: irsa-demo-sa
namespace: default
resourceVersion: "208932"
uid: 4c6298b9-d888-4212-8a71-67a242b24b1f
secrets:
- name: irsa-demo-sa-token-vdstw
Tiếp đến là mình tạo Job rồi gắn service account vào
Cuối cùng là run lệnh s3 ls sem có list được ss3 trên aws ko?
# Resource: Kubernetes Job
resource "kubernetes_job_v1" "irsa_demo" {
metadata {
name = "irsa-demo"
}
spec {
template {
metadata {
labels = {
app = "irsa-demo"
}
}
spec {
service_account_name = kubernetes_service_account_v1.irsa_demo_sa.metadata.0.name
container {
name = "irsa-demo"
image = "amazon/aws-cli:latest"
args = ["s3", "ls"]
#args = ["ec2", "describe-instances", "--region", "${var.aws_region}"] # Should fail as we don't have access to EC2 Describe Instances for IAM Role
}
restart_policy = "Never"
}
}
}
}
Giờ mình tiến hành list job, pod và show logs pods.
root@LP11-D7891:~# kubectl get job NAME COMPLETIONS DURATION AGE irsa-demo 1/1 11s 14s root@LP11-D7891:~# kubectl get pod NAME READY STATUS RESTARTS AGE irsa-demo-dsjzr 0/1 Completed 0 44s root@LP11-D7891:~# kubectl logs pod/irsa-demo-dsjzr 2022-09-04 16:46:10 codepipeline-us-east-1-95999394200 2022-07-12 16:44:47 elasticbeanstalk-eu-west-1-250887682577 2022-09-07 09:14:06 terraform-on-aws-eks-nim
3) Use eks-irsa terraform module to create IRSA easily
https://registry.terraform.io/modules/mrnim94/eks-irsa/aws/latest
4) Use cross-account IAM roles in IRSA
4.1) Try hard.
Cách này chúng ta hơi manual 1 tý.
Pods của cluster A muốn access vào resource AWS của account B
Đầu tiền bạn cần lấy OpenID Connect Provider của EKS trong accunt A.https://oidc.eks.us-west-2.amazonaws.com/id/AAAAAAAAAAAAAAAAAA
Tiếp đến bạn run terraform bên dưới ở account của Cluster B.
resource "aws_iam_openid_connect_provider" "oidc_issuer" {
url = "https://oidc.eks.us-west-2.amazonaws.com/id/AAAAAAAAAAAAAAAAAA"
thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280"]
client_id_list = ["sts.amazonaws.com"]
}
data "aws_iam_policy_document" "irsa" {
statement {
effect = "Allow"
actions = ["sts:AssumeRoleWithWebIdentity"]
principals {
type = "Federated"
identifiers = [aws_iam_openid_connect_provider.oidc_issuer.arn]
}
condition {
test = "StringLike"
variable = "oidc.eks.us-west-2.amazonaws.com/id/AAAAAAAAAAAAAAAAAA:sub"
values = ["system:serviceaccount:kube-system:zeus-rotations"]
}
}
}
data "aws_iam_policy_document" "iam" {
statement {
effect = "Allow"
actions = [
"iam:CreateAccessKey",
"iam:ListAccessKeys",
"iam:DeleteAccessKey",
]
resources = ["*"]
}
}
resource "aws_iam_role" "iam" {
name = "irsa"
assume_role_policy = data.aws_iam_policy_document.irsa.json
}
resource "aws_iam_policy" "iam" {
name = "iam"
policy = data.aws_iam_policy_document.iam.json
}
resource "aws_iam_role_policy_attachment" "s3" {
role = aws_iam_role.iam.name
policy_arn = aws_iam_policy.iam.arn
}
output "irsa_iam_role_arn" {
description = "IRSA Demo IAM Role ARN. You need add this values to serviceAccount.annotations with key `eks.amazonaws.com/role-arn`"
value = aws_iam_role.iam.arn
}
Khi nó print cái output irsa_iam_role_arn thì bạn set nó vào service account của deployment của aws account A.
4.2) Using IRSA Module.
Cũng như các bước trên bạn cần lấy được https://oidc.eks.us-west-2.amazonaws.com/id/AAAAAAAAAAAAAAAAAA
resource "aws_iam_openid_connect_provider" "oidc_issuer" {
url = "https://oidc.eks.us-west-2.amazonaws.com/id/AAAAAAAAAAAAAAAAAA"
thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280"]
client_id_list = ["sts.amazonaws.com"]
}
module "eks-irsa-nim-devsecops" {
source = "mrnim94/eks-irsa/aws"
version = "1.0.0"
depends_on = [
aws_iam_openid_connect_provider.oidc_issuer
]
aws_region = var.aws_region
environment = "dev"
business_divsion = "irsa-zeus-rotations-nim-devsecops"
k8s_namespace = "kube-system"
k8s_service_account = "zeus-rotations"
json_policy = <<EOT
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iam:CreateAccessKey",
"iam:ListAccessKeys",
"iam:DeleteAccessKey"
],
"Resource": "*"
}
]
}
EOT
aws_iam_openid_connect_provider_arn = aws_iam_openid_connect_provider.oidc_issuer.arn
}
output "irsa_iam_role_arn_nim_devsecops" {
description = "aws_iam_openid_connect_provider_arn"
value = module.eks-irsa-nim-devsecops.irsa_iam_role_arn
}
